From ecb6fd5801a93dadc992b701a97fc7af89689bb4 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 1 Aug 2023 11:29:40 +0200 Subject: [PATCH 001/180] Nuke CameraX --- android/build.gradle | 12 +- .../com/mrousavy/camera/CameraView+Focus.kt | 24 +- .../mrousavy/camera/CameraView+RecordVideo.kt | 88 +---- .../mrousavy/camera/CameraView+TakePhoto.kt | 99 +---- .../camera/CameraView+TakeSnapshot.kt | 53 +-- .../java/com/mrousavy/camera/CameraView.kt | 353 ++---------------- .../com/mrousavy/camera/CameraViewManager.kt | 7 - .../com/mrousavy/camera/CameraViewModule.kt | 6 +- .../main/java/com/mrousavy/camera/Errors.kt | 1 - .../mrousavy/camera/frameprocessor/Frame.java | 51 ++- .../com/mrousavy/camera/parsers/Size+easy.kt | 3 - .../com/mrousavy/camera/utils/AspectRatio.kt | 28 -- .../com/mrousavy/camera/utils/CameraDevice.kt | 11 +- .../camera/utils/CameraSelector+byID.kt | 25 -- .../com/mrousavy/camera/utils/DeviceFormat.kt | 33 -- .../utils/ExifInterface+buildMetadataMap.kt | 62 --- .../camera/utils/ImageCapture+suspendables.kt | 41 -- .../mrousavy/camera/utils/ImageProxy+isRaw.kt | 12 - .../mrousavy/camera/utils/ImageProxy+save.kt | 127 ------- .../example/ExampleFrameProcessorPlugin.java | 5 +- example/ios/Podfile.lock | 2 +- 21 files changed, 76 insertions(+), 967 deletions(-) delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/AspectRatio.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/CameraSelector+byID.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/DeviceFormat.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/ExifInterface+buildMetadataMap.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/ImageCapture+suspendables.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/ImageProxy+isRaw.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/ImageProxy+save.kt diff --git a/android/build.gradle b/android/build.gradle index c66ae9ab5b..5ebb757189 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -142,21 +142,11 @@ dependencies { //noinspection GradleDynamicVersion implementation 'com.facebook.react:react-android:+' - implementation 'androidx.core:core-ktx:1.3.2' + implementation "androidx.core:core-ktx:1.3.2" implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.5.2" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2" - implementation "androidx.camera:camera-core:1.1.0" - implementation "androidx.camera:camera-camera2:1.1.0" - implementation "androidx.camera:camera-lifecycle:1.1.0" - implementation "androidx.camera:camera-video:1.1.0" - - implementation "androidx.camera:camera-view:1.1.0" - implementation "androidx.camera:camera-extensions:1.1.0" - - implementation "androidx.exifinterface:exifinterface:1.3.3" - implementation project(":react-native-worklets") implementation project(":shopify_react-native-skia") } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt index 0a13fa3540..0c17daddef 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt @@ -1,29 +1,7 @@ package com.mrousavy.camera -import androidx.camera.core.FocusMeteringAction import com.facebook.react.bridge.ReadableMap -import kotlinx.coroutines.guava.await -import kotlinx.coroutines.withContext -import java.util.concurrent.TimeUnit suspend fun CameraView.focus(pointMap: ReadableMap) { - val cameraControl = camera?.cameraControl ?: throw CameraNotReadyError() - if (!pointMap.hasKey("x") || !pointMap.hasKey("y")) { - throw InvalidTypeScriptUnionError("point", pointMap.toString()) - } - - val dpi = resources.displayMetrics.density - val x = pointMap.getDouble("x") * dpi - val y = pointMap.getDouble("y") * dpi - - // Getting the point from the previewView needs to be run on the UI thread - val point = withContext(coroutineScope.coroutineContext) { - previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat()) - } - - val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE) - .setAutoCancelDuration(5, TimeUnit.SECONDS) // auto-reset after 5 seconds - .build() - - cameraControl.startFocusAndMetering(action).await() + // TODO: CameraView.focus!! } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt index 14ee747215..eb27ef51c6 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt @@ -3,27 +3,13 @@ package com.mrousavy.camera import android.Manifest import android.annotation.SuppressLint import android.content.pm.PackageManager -import androidx.camera.video.FileOutputOptions -import androidx.camera.video.VideoRecordEvent import androidx.core.content.ContextCompat -import androidx.core.util.Consumer import com.facebook.react.bridge.* -import com.mrousavy.camera.utils.makeErrorMap import java.io.File import java.text.SimpleDateFormat import java.util.* -data class TemporaryFile(val path: String) - fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback) { - if (videoCapture == null) { - if (video == true) { - throw CameraNotReadyError() - } else { - throw VideoNotEnabledError() - } - } - // check audio permission if (audio == true) { if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { @@ -34,89 +20,27 @@ fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback) if (options.hasKey("flash")) { val enableFlash = options.getString("flash") == "on" // overrides current torch mode value to enable flash while recording - camera!!.cameraControl.enableTorch(enableFlash) + // TODO: Enable torch for flash } val id = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date()) val file = File.createTempFile("VisionCamera-${id}", ".mp4") - val fileOptions = FileOutputOptions.Builder(file).build() - - val recorder = videoCapture!!.output - var recording = recorder.prepareRecording(context, fileOptions) - if (audio == true) { - @SuppressLint("MissingPermission") - recording = recording.withAudioEnabled() - } - - activeVideoRecording = recording.start(ContextCompat.getMainExecutor(context), object : Consumer { - override fun accept(event: VideoRecordEvent?) { - if (event is VideoRecordEvent.Finalize) { - if (event.hasError()) { - // error occured! - val error = when (event.error) { - VideoRecordEvent.Finalize.ERROR_ENCODING_FAILED -> VideoEncoderError(event.cause) - VideoRecordEvent.Finalize.ERROR_FILE_SIZE_LIMIT_REACHED -> FileSizeLimitReachedError(event.cause) - VideoRecordEvent.Finalize.ERROR_INSUFFICIENT_STORAGE -> InsufficientStorageError(event.cause) - VideoRecordEvent.Finalize.ERROR_INVALID_OUTPUT_OPTIONS -> InvalidVideoOutputOptionsError(event.cause) - VideoRecordEvent.Finalize.ERROR_NO_VALID_DATA -> NoValidDataError(event.cause) - VideoRecordEvent.Finalize.ERROR_RECORDER_ERROR -> RecorderError(event.cause) - VideoRecordEvent.Finalize.ERROR_SOURCE_INACTIVE -> InactiveSourceError(event.cause) - else -> UnknownCameraError(event.cause) - } - val map = makeErrorMap("${error.domain}/${error.id}", error.message, error) - onRecordCallback(null, map) - } else { - // recording saved successfully! - val map = Arguments.createMap() - map.putString("path", event.outputResults.outputUri.toString()) - map.putDouble("duration", /* seconds */ event.recordingStats.recordedDurationNanos.toDouble() / 1000000.0 / 1000.0) - map.putDouble("size", /* kB */ event.recordingStats.numBytesRecorded.toDouble() / 1000.0) - onRecordCallback(map, null) - } - - // reset the torch mode - camera!!.cameraControl.enableTorch(torch == "on") - } - } - }) + // TODO: startRecording() } @SuppressLint("RestrictedApi") fun CameraView.pauseRecording() { - if (videoCapture == null) { - throw CameraNotReadyError() - } - if (activeVideoRecording == null) { - throw NoRecordingInProgressError() - } - - activeVideoRecording!!.pause() + // TODO: pauseRecording() } @SuppressLint("RestrictedApi") fun CameraView.resumeRecording() { - if (videoCapture == null) { - throw CameraNotReadyError() - } - if (activeVideoRecording == null) { - throw NoRecordingInProgressError() - } - - activeVideoRecording!!.resume() + // TODO: resumeRecording() } @SuppressLint("RestrictedApi") fun CameraView.stopRecording() { - if (videoCapture == null) { - throw CameraNotReadyError() - } - if (activeVideoRecording == null) { - throw NoRecordingInProgressError() - } - - activeVideoRecording!!.stop() - - // reset torch mode to original value - camera!!.cameraControl.enableTorch(torch == "on") + // TODO: stopRecording() + // TODO: disable torch again } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index cb7854e49e..e40e08aad0 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -2,113 +2,16 @@ package com.mrousavy.camera import android.annotation.SuppressLint import android.hardware.camera2.* -import android.util.Log -import androidx.camera.camera2.interop.Camera2CameraInfo -import androidx.camera.core.ImageCapture -import androidx.camera.core.ImageProxy -import androidx.exifinterface.media.ExifInterface import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap import com.mrousavy.camera.utils.* import kotlinx.coroutines.* -import java.io.File -import kotlin.system.measureTimeMillis @SuppressLint("UnsafeOptInUsageError") suspend fun CameraView.takePhoto(options: ReadableMap): WritableMap = coroutineScope { - if (fallbackToSnapshot) { - Log.i(CameraView.TAG, "takePhoto() called, but falling back to Snapshot because 1 use-case is already occupied.") - return@coroutineScope takeSnapshot(options) - } - - val startFunc = System.nanoTime() - Log.i(CameraView.TAG, "takePhoto() called") - if (imageCapture == null) { - if (photo == true) { - throw CameraNotReadyError() - } else { - throw PhotoNotEnabledError() - } - } - - if (options.hasKey("flash")) { - val flashMode = options.getString("flash") - imageCapture!!.flashMode = when (flashMode) { - "on" -> ImageCapture.FLASH_MODE_ON - "off" -> ImageCapture.FLASH_MODE_OFF - "auto" -> ImageCapture.FLASH_MODE_AUTO - else -> throw InvalidTypeScriptUnionError("flash", flashMode ?: "(null)") - } - } - // All those options are not yet implemented - see https://github.com/mrousavy/react-native-vision-camera/issues/75 - if (options.hasKey("photoCodec")) { - // TODO photoCodec - } - if (options.hasKey("qualityPrioritization")) { - // TODO qualityPrioritization - } - if (options.hasKey("enableAutoRedEyeReduction")) { - // TODO enableAutoRedEyeReduction - } - if (options.hasKey("enableDualCameraFusion")) { - // TODO enableDualCameraFusion - } - if (options.hasKey("enableAutoStabilization")) { - // TODO enableAutoStabilization - } - if (options.hasKey("enableAutoDistortionCorrection")) { - // TODO enableAutoDistortionCorrection - } - val skipMetadata = if (options.hasKey("skipMetadata")) options.getBoolean("skipMetadata") else false - - val camera2Info = Camera2CameraInfo.from(camera!!.cameraInfo) - val lensFacing = camera2Info.getCameraCharacteristic(CameraCharacteristics.LENS_FACING) - - val results = awaitAll( - async(coroutineContext) { - Log.d(CameraView.TAG, "Taking picture...") - val startCapture = System.nanoTime() - val pic = imageCapture!!.takePicture(takePhotoExecutor) - val endCapture = System.nanoTime() - Log.i(CameraView.TAG_PERF, "Finished image capture in ${(endCapture - startCapture) / 1_000_000}ms") - pic - }, - async(Dispatchers.IO) { - Log.d(CameraView.TAG, "Creating temp file...") - File.createTempFile("mrousavy", ".jpg", context.cacheDir).apply { deleteOnExit() } - } - ) - val photo = results.first { it is ImageProxy } as ImageProxy - val file = results.first { it is File } as File - - val exif: ExifInterface? - @Suppress("BlockingMethodInNonBlockingContext") - withContext(Dispatchers.IO) { - Log.d(CameraView.TAG, "Saving picture to ${file.absolutePath}...") - val milliseconds = measureTimeMillis { - val flipHorizontally = lensFacing == CameraCharacteristics.LENS_FACING_FRONT - photo.save(file, flipHorizontally) - } - Log.i(CameraView.TAG_PERF, "Finished image saving in ${milliseconds}ms") - // TODO: Read Exif from existing in-memory photo buffer instead of file? - exif = if (skipMetadata) null else ExifInterface(file) - } + // TODO: takePhoto() val map = Arguments.createMap() - map.putString("path", file.absolutePath) - map.putInt("width", photo.width) - map.putInt("height", photo.height) - map.putBoolean("isRawPhoto", photo.isRaw) - - val metadata = exif?.buildMetadataMap() - map.putMap("metadata", metadata) - - photo.close() - - Log.d(CameraView.TAG, "Finished taking photo!") - - val endFunc = System.nanoTime() - Log.i(CameraView.TAG_PERF, "Finished function execution in ${(endFunc - startFunc) / 1_000_000}ms") return@coroutineScope map } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt index bb40925412..2c34123fdd 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt @@ -1,60 +1,13 @@ package com.mrousavy.camera -import android.graphics.Bitmap -import androidx.exifinterface.media.ExifInterface import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap -import com.mrousavy.camera.utils.buildMetadataMap -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.withContext -import java.io.File -import java.io.FileOutputStream -import kotlinx.coroutines.guava.await suspend fun CameraView.takeSnapshot(options: ReadableMap): WritableMap = coroutineScope { - val camera = camera ?: throw CameraNotReadyError() - val enableFlash = options.getString("flash") == "on" + // TODO: takeSnapshot() - try { - if (enableFlash) { - camera.cameraControl.enableTorch(true).await() - } - - val bitmap = withContext(coroutineScope.coroutineContext) { - previewView.bitmap ?: throw CameraNotReadyError() - } - - val quality = if (options.hasKey("quality")) options.getInt("quality") else 100 - - val file: File - val exif: ExifInterface - @Suppress("BlockingMethodInNonBlockingContext") - withContext(Dispatchers.IO) { - file = File.createTempFile("mrousavy", ".jpg", context.cacheDir).apply { deleteOnExit() } - FileOutputStream(file).use { stream -> - bitmap.compress(Bitmap.CompressFormat.JPEG, quality, stream) - } - exif = ExifInterface(file) - } - - val map = Arguments.createMap() - map.putString("path", file.absolutePath) - map.putInt("width", bitmap.width) - map.putInt("height", bitmap.height) - map.putBoolean("isRawPhoto", false) - - val skipMetadata = - if (options.hasKey("skipMetadata")) options.getBoolean("skipMetadata") else false - val metadata = if (skipMetadata) null else exif.buildMetadataMap() - map.putMap("metadata", metadata) - - return@coroutineScope map - } finally { - if (enableFlash) { - // reset to `torch` property - camera.cameraControl.enableTorch(this@takeSnapshot.torch == "on") - } - } + val map = Arguments.createMap() + return@coroutineScope map } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 5cc7654c77..d26c84a458 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -7,33 +7,16 @@ import android.content.pm.PackageManager import android.content.res.Configuration import android.hardware.camera2.* import android.util.Log -import android.util.Range import android.view.* -import android.view.View.OnTouchListener import android.widget.FrameLayout -import androidx.camera.camera2.interop.Camera2Interop -import androidx.camera.core.* -import androidx.camera.core.impl.* -import androidx.camera.extensions.* -import androidx.camera.lifecycle.ProcessCameraProvider -import androidx.camera.video.* -import androidx.camera.video.VideoCapture -import androidx.camera.view.PreviewView import androidx.core.content.ContextCompat import androidx.lifecycle.* -import com.facebook.jni.HybridData -import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.* -import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor -import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin -import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry import com.mrousavy.camera.utils.* import kotlinx.coroutines.* -import kotlinx.coroutines.guava.await import java.lang.IllegalArgumentException import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors import kotlin.math.max import kotlin.math.min @@ -67,7 +50,7 @@ import kotlin.math.min @Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that. @SuppressLint("ClickableViewAccessibility", "ViewConstructor") -class CameraView(context: Context, private val frameProcessorThread: ExecutorService) : FrameLayout(context), LifecycleOwner { +class CameraView(context: Context, private val frameProcessorThread: ExecutorService) : FrameLayout(context) { companion object { const val TAG = "CameraView" const val TAG_PERF = "CameraView.performance" @@ -98,41 +81,11 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer var torch = "off" var zoom: Float = 1f // in "factor" var orientation: String? = null - var enableZoomGesture = false - set(value) { - field = value - setOnTouchListener(if (value) touchEventListener else null) - } // private properties private var isMounted = false - private val reactContext: ReactContext - get() = context as ReactContext - - @Suppress("JoinDeclarationAndAssignment") - internal val previewView: PreviewView - private val cameraExecutor = Executors.newSingleThreadExecutor() - internal val takePhotoExecutor = Executors.newSingleThreadExecutor() - internal val recordVideoExecutor = Executors.newSingleThreadExecutor() - internal var coroutineScope = CoroutineScope(Dispatchers.Main) - internal var camera: Camera? = null - internal var imageCapture: ImageCapture? = null - internal var videoCapture: VideoCapture? = null public var frameProcessor: FrameProcessor? = null - private var preview: Preview? = null - private var imageAnalysis: ImageAnalysis? = null - - internal var activeVideoRecording: Recording? = null - - private var extensionsManager: ExtensionsManager? = null - - private val scaleGestureListener: ScaleGestureDetector.SimpleOnScaleGestureListener - private val scaleGestureDetector: ScaleGestureDetector - private val touchEventListener: OnTouchListener - - private val lifecycleRegistry: LifecycleRegistry - private var hostLifecycleState: Lifecycle.State private val inputRotation: Int get() { @@ -158,115 +111,17 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer private var minZoom: Float = 1f private var maxZoom: Float = 1f - @Suppress("RedundantIf") - internal val fallbackToSnapshot: Boolean - @SuppressLint("UnsafeOptInUsageError") - get() { - if (video != true && !enableFrameProcessor) { - // Both use-cases are disabled, so `photo` is the only use-case anyways. Don't need to fallback here. - return false - } - cameraId?.let { cameraId -> - val cameraManger = reactContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager - cameraManger?.let { - val characteristics = cameraManger.getCameraCharacteristics(cameraId) - val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) - if (hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) { - // Camera only supports a single use-case at a time - return true - } else { - if (video == true && enableFrameProcessor) { - // Camera supports max. 2 use-cases, but both are occupied by `frameProcessor` and `video` - return true - } else { - // Camera supports max. 2 use-cases and only one is occupied (either `frameProcessor` or `video`), so we can add `photo` - return false - } - } - } - } - return false - } - init { - previewView = PreviewView(context) - previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) - previewView.installHierarchyFitter() // If this is not called correctly, view finder will be black/blank - addView(previewView) - - scaleGestureListener = object : ScaleGestureDetector.SimpleOnScaleGestureListener() { - override fun onScale(detector: ScaleGestureDetector): Boolean { - zoom = max(min((zoom * detector.scaleFactor), maxZoom), minZoom) - update(arrayListOfZoom) - return true - } - } - scaleGestureDetector = ScaleGestureDetector(context, scaleGestureListener) - touchEventListener = OnTouchListener { _, event -> return@OnTouchListener scaleGestureDetector.onTouchEvent(event) } - - hostLifecycleState = Lifecycle.State.INITIALIZED - lifecycleRegistry = LifecycleRegistry(this) - reactContext.addLifecycleEventListener(object : LifecycleEventListener { - override fun onHostResume() { - hostLifecycleState = Lifecycle.State.RESUMED - updateLifecycleState() - // workaround for https://issuetracker.google.com/issues/147354615, preview must be bound on resume - update(propsThatRequireSessionReconfiguration) - } - override fun onHostPause() { - hostLifecycleState = Lifecycle.State.CREATED - updateLifecycleState() - } - override fun onHostDestroy() { - hostLifecycleState = Lifecycle.State.DESTROYED - updateLifecycleState() - cameraExecutor.shutdown() - takePhotoExecutor.shutdown() - recordVideoExecutor.shutdown() - reactContext.removeLifecycleEventListener(this) - } - }) } override fun onConfigurationChanged(newConfig: Configuration?) { super.onConfigurationChanged(newConfig) - updateOrientation() - } - - @SuppressLint("RestrictedApi") - private fun updateOrientation() { - preview?.targetRotation = inputRotation - imageCapture?.targetRotation = outputRotation - videoCapture?.targetRotation = outputRotation - imageAnalysis?.targetRotation = outputRotation - } - - override fun getLifecycle(): Lifecycle { - return lifecycleRegistry - } - - /** - * Updates the custom Lifecycle to match the host activity's lifecycle, and if it's active we narrow it down to the [isActive] and [isAttachedToWindow] fields. - */ - private fun updateLifecycleState() { - val lifecycleBefore = lifecycleRegistry.currentState - if (hostLifecycleState == Lifecycle.State.RESUMED) { - // Host Lifecycle (Activity) is currently active (RESUMED), so we narrow it down to the view's lifecycle - if (isActive && isAttachedToWindow) { - lifecycleRegistry.currentState = Lifecycle.State.RESUMED - } else { - lifecycleRegistry.currentState = Lifecycle.State.CREATED - } - } else { - // Host Lifecycle (Activity) is currently inactive (STARTED or DESTROYED), so that overrules our view's lifecycle - lifecycleRegistry.currentState = hostLifecycleState - } - Log.d(TAG, "Lifecycle went from ${lifecycleBefore.name} -> ${lifecycleRegistry.currentState.name} (isActive: $isActive | isAttachedToWindow: $isAttachedToWindow)") + // TODO: updateOrientation() } override fun onAttachedToWindow() { super.onAttachedToWindow() - updateLifecycleState() + // TODO: updateLifecycleState() if (!isMounted) { isMounted = true invokeOnViewReady() @@ -275,51 +130,45 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer override fun onDetachedFromWindow() { super.onDetachedFromWindow() - updateLifecycleState() + // TODO: updateLifecycleState() } /** * Invalidate all React Props and reconfigure the device */ - fun update(changedProps: ArrayList) = previewView.post { - // TODO: Does this introduce too much overhead? - // I need to .post on the previewView because it might've not been initialized yet - // I need to use CoroutineScope.launch because of the suspend fun [configureSession] - coroutineScope.launch { - try { - val shouldReconfigureSession = changedProps.containsAny(propsThatRequireSessionReconfiguration) - val shouldReconfigureZoom = shouldReconfigureSession || changedProps.contains("zoom") - val shouldReconfigureTorch = shouldReconfigureSession || changedProps.contains("torch") - val shouldUpdateOrientation = shouldReconfigureSession || changedProps.contains("orientation") + fun update(changedProps: ArrayList) { + try { + val shouldReconfigureSession = changedProps.containsAny(propsThatRequireSessionReconfiguration) + val shouldReconfigureZoom = shouldReconfigureSession || changedProps.contains("zoom") + val shouldReconfigureTorch = shouldReconfigureSession || changedProps.contains("torch") + val shouldUpdateOrientation = shouldReconfigureSession || changedProps.contains("orientation") - if (changedProps.contains("isActive")) { - updateLifecycleState() - } - if (shouldReconfigureSession) { - configureSession() - } - if (shouldReconfigureZoom) { - val zoomClamped = max(min(zoom, maxZoom), minZoom) - camera!!.cameraControl.setZoomRatio(zoomClamped) - } - if (shouldReconfigureTorch) { - camera!!.cameraControl.enableTorch(torch == "on") - } - if (shouldUpdateOrientation) { - updateOrientation() - } - } catch (e: Throwable) { - Log.e(TAG, "update() threw: ${e.message}") - invokeOnError(e) + if (changedProps.contains("isActive")) { + // TODO: updateLifecycleState() + } + if (shouldReconfigureSession) { + configureSession() } + if (shouldReconfigureZoom) { + val zoomClamped = max(min(zoom, maxZoom), minZoom) + // TODO: camera!!.cameraControl.setZoomRatio(zoomClamped) + } + if (shouldReconfigureTorch) { + // TODO: camera!!.cameraControl.enableTorch(torch == "on") + } + if (shouldUpdateOrientation) { + // TODO: updateOrientation() + } + } catch (e: Throwable) { + Log.e(TAG, "update() threw: ${e.message}") + invokeOnError(e) } } /** * Configures the camera capture session. This should only be called when the camera device changes. */ - @SuppressLint("RestrictedApi", "UnsafeOptInUsageError") - private suspend fun configureSession() { + private fun configureSession() { try { val startTime = System.currentTimeMillis() Log.i(TAG, "Configuring session...") @@ -329,150 +178,12 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer if (cameraId == null) { throw NoCameraDeviceError() } - if (format != null) - Log.i(TAG, "Configuring session with Camera ID $cameraId and custom format...") - else - Log.i(TAG, "Configuring session with Camera ID $cameraId and default format options...") - - // Used to bind the lifecycle of cameras to the lifecycle owner - val cameraProvider = ProcessCameraProvider.getInstance(reactContext).await() - - var cameraSelector = CameraSelector.Builder().byID(cameraId!!).build() - - val tryEnableExtension: (suspend (extension: Int) -> Unit) = lambda@ { extension -> - if (extensionsManager == null) { - Log.i(TAG, "Initializing ExtensionsManager...") - extensionsManager = ExtensionsManager.getInstanceAsync(context, cameraProvider).await() - } - if (extensionsManager!!.isExtensionAvailable(cameraSelector, extension)) { - Log.i(TAG, "Enabling extension $extension...") - cameraSelector = extensionsManager!!.getExtensionEnabledCameraSelector(cameraSelector, extension) - } else { - Log.e(TAG, "Extension $extension is not available for the given Camera!") - throw when (extension) { - ExtensionMode.HDR -> HdrNotContainedInFormatError() - ExtensionMode.NIGHT -> LowLightBoostNotContainedInFormatError() - else -> Error("Invalid extension supplied! Extension $extension is not available.") - } - } - } - - val previewBuilder = Preview.Builder() - .setTargetRotation(inputRotation) - - val imageCaptureBuilder = ImageCapture.Builder() - .setTargetRotation(outputRotation) - .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY) - - val videoRecorderBuilder = Recorder.Builder() - .setExecutor(cameraExecutor) - - val imageAnalysisBuilder = ImageAnalysis.Builder() - .setTargetRotation(outputRotation) - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) - .setBackgroundExecutor(frameProcessorThread) - - if (format == null) { - // let CameraX automatically find best resolution for the target aspect ratio - Log.i(TAG, "No custom format has been set, CameraX will automatically determine best configuration...") - val aspectRatio = aspectRatio(previewView.height, previewView.width) // flipped because it's in sensor orientation. - previewBuilder.setTargetAspectRatio(aspectRatio) - imageCaptureBuilder.setTargetAspectRatio(aspectRatio) - // TODO: Aspect Ratio for Video Recorder? - imageAnalysisBuilder.setTargetAspectRatio(aspectRatio) - } else { - // User has selected a custom format={}. Use that - val format = DeviceFormat(format!!) - Log.i(TAG, "Using custom format - photo: ${format.photoSize}, video: ${format.videoSize} @ $fps FPS") - if (video == true) { - previewBuilder.setTargetResolution(format.videoSize) - } else { - previewBuilder.setTargetResolution(format.photoSize) - } - imageCaptureBuilder.setTargetResolution(format.photoSize) - imageAnalysisBuilder.setTargetResolution(format.photoSize) - - // TODO: Ability to select resolution exactly depending on format? Just like on iOS... - when (min(format.videoSize.height, format.videoSize.width)) { - in 0..480 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.SD)) - in 480..720 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.HD, FallbackStrategy.lowerQualityThan(Quality.HD))) - in 720..1080 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.FHD, FallbackStrategy.lowerQualityThan(Quality.FHD))) - in 1080..2160 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.UHD, FallbackStrategy.lowerQualityThan(Quality.UHD))) - in 2160..4320 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.HIGHEST, FallbackStrategy.lowerQualityThan(Quality.HIGHEST))) - } - - fps?.let { fps -> - if (format.frameRateRanges.any { it.contains(fps) }) { - // Camera supports the given FPS (frame rate range) - val frameDuration = (1.0 / fps.toDouble()).toLong() * 1_000_000_000 - - Log.i(TAG, "Setting AE_TARGET_FPS_RANGE to $fps-$fps, and SENSOR_FRAME_DURATION to $frameDuration") - Camera2Interop.Extender(previewBuilder) - .setCaptureRequestOption(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) - .setCaptureRequestOption(CaptureRequest.SENSOR_FRAME_DURATION, frameDuration) - // TODO: Frame Rate/FPS for Video Recorder? - } else { - throw FpsNotContainedInFormatError(fps) - } - } - if (hdr == true) { - tryEnableExtension(ExtensionMode.HDR) - } - if (lowLightBoost == true) { - tryEnableExtension(ExtensionMode.NIGHT) - } - } - - - // Unbind use cases before rebinding - videoCapture = null - imageCapture = null - imageAnalysis = null - cameraProvider.unbindAll() - - // Bind use cases to camera - val useCases = ArrayList() - if (video == true) { - Log.i(TAG, "Adding VideoCapture use-case...") - - val videoRecorder = videoRecorderBuilder.build() - videoCapture = VideoCapture.withOutput(videoRecorder) - videoCapture!!.targetRotation = outputRotation - useCases.add(videoCapture!!) - } - if (photo == true) { - if (fallbackToSnapshot) { - Log.i(TAG, "Tried to add photo use-case (`photo={true}`) but the Camera device only supports " + - "a single use-case at a time. Falling back to Snapshot capture.") - } else { - Log.i(TAG, "Adding ImageCapture use-case...") - imageCapture = imageCaptureBuilder.build() - useCases.add(imageCapture!!) - } - } - if (enableFrameProcessor) { - Log.i(TAG, "Adding ImageAnalysis use-case...") - imageAnalysis = imageAnalysisBuilder.build().apply { - setAnalyzer(cameraExecutor) { image -> - // Call JS Frame Processor - val frame = Frame(image) - frameProcessor?.call(frame) - // ...frame gets closed in FrameHostObject implementation via JS ref counting - } - } - useCases.add(imageAnalysis!!) - } - - preview = previewBuilder.build() - Log.i(TAG, "Attaching ${useCases.size} use-cases...") - camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, *useCases.toTypedArray()) - preview!!.setSurfaceProvider(previewView.surfaceProvider) - minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f - maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f + // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f + // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f val duration = System.currentTimeMillis() - startTime - Log.i(TAG_PERF, "Session configured in $duration ms! Camera: ${camera!!}") + Log.i(TAG_PERF, "Session configured in $duration ms!") invokeOnInitialized() } catch (exc: Throwable) { Log.e(TAG, "Failed to configure session: ${exc.message}") diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt index cdd6232f4c..383ee66930 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt @@ -150,13 +150,6 @@ class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManage view.zoom = zoomFloat } - @ReactProp(name = "enableZoomGesture") - fun setEnableZoomGesture(view: CameraView, enableZoomGesture: Boolean) { - if (view.enableZoomGesture != enableZoomGesture) - addChangedPropToTransaction(view, "enableZoomGesture") - view.enableZoomGesture = enableZoomGesture - } - @ReactProp(name = "orientation") fun setOrientation(view: CameraView, orientation: String) { if (view.orientation != orientation) diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index 5a7a8750c3..27d110d256 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -6,8 +6,6 @@ import android.content.pm.PackageManager import android.hardware.camera2.CameraManager import android.os.Build import android.util.Log -import androidx.camera.extensions.ExtensionsManager -import androidx.camera.lifecycle.ProcessCameraProvider import androidx.core.content.ContextCompat import com.facebook.react.bridge.* import com.facebook.react.module.annotations.ReactModule @@ -146,13 +144,11 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ fun getAvailableCameraDevices(promise: Promise) { coroutineScope.launch { withPromise(promise) { - val cameraProvider = ProcessCameraProvider.getInstance(reactApplicationContext).await() - val extensionsManager = ExtensionsManager.getInstanceAsync(reactApplicationContext, cameraProvider).await() val manager = reactApplicationContext.getSystemService(Context.CAMERA_SERVICE) as CameraManager val devices = Arguments.createArray() manager.cameraIdList.forEach { cameraId -> - val device = CameraDevice(manager, extensionsManager, cameraId) + val device = CameraDevice(manager, cameraId) devices.pushMap(device.toMap()) } promise.resolve(devices) diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index eae2bf63b6..2147e756b9 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -1,7 +1,6 @@ package com.mrousavy.camera import android.graphics.ImageFormat -import androidx.camera.video.VideoRecordEvent.Finalize.VideoRecordError abstract class CameraError( /** diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index d5ccb2edc6..a09ca53ebf 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -1,43 +1,43 @@ package com.mrousavy.camera.frameprocessor; -import android.annotation.SuppressLint; import android.graphics.ImageFormat; -import android.graphics.Matrix; import android.media.Image; -import androidx.camera.core.ImageProxy; import com.facebook.proguard.annotations.DoNotStrip; import java.nio.ByteBuffer; public class Frame { - private final ImageProxy imageProxy; - - public Frame(ImageProxy imageProxy) { - this.imageProxy = imageProxy; + private final Image image; + private final boolean isMirrored; + private final long timestamp; + private final int orientation; + + public Frame(Image image, long timestamp, int orientation, boolean isMirrored) { + this.image = image; + this.timestamp = timestamp; + this.orientation = orientation; + this.isMirrored = isMirrored; } - public ImageProxy getImageProxy() { - return imageProxy; + public Image getImage() { + return image; } @SuppressWarnings("unused") @DoNotStrip public int getWidth() { - return imageProxy.getWidth(); + return image.getWidth(); } @SuppressWarnings("unused") @DoNotStrip public int getHeight() { - return imageProxy.getHeight(); + return image.getHeight(); } @SuppressWarnings("unused") @DoNotStrip public boolean getIsValid() { try { - @SuppressLint("UnsafeOptInUsageError") - Image image = imageProxy.getImage(); - if (image == null) return false; // will throw an exception if the image is already closed image.getCropRect(); // no exception thrown, image must still be valid. @@ -51,21 +51,20 @@ public boolean getIsValid() { @SuppressWarnings("unused") @DoNotStrip public boolean getIsMirrored() { - Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); - // TODO: Figure out how to get isMirrored from ImageProxy - return false; + return isMirrored; } @SuppressWarnings("unused") @DoNotStrip public long getTimestamp() { - return imageProxy.getImageInfo().getTimestamp(); + return timestamp; } @SuppressWarnings("unused") @DoNotStrip public String getOrientation() { - int rotation = imageProxy.getImageInfo().getRotationDegrees(); + // TODO: Check if this works as expected + int rotation = orientation; if (rotation >= 45 && rotation < 135) return "landscapeRight"; if (rotation >= 135 && rotation < 225) @@ -78,13 +77,13 @@ public String getOrientation() { @SuppressWarnings("unused") @DoNotStrip public int getPlanesCount() { - return imageProxy.getPlanes().length; + return image.getPlanes().length; } @SuppressWarnings("unused") @DoNotStrip public int getBytesPerRow() { - return imageProxy.getPlanes()[0].getRowStride(); + return image.getPlanes()[0].getRowStride(); } private static byte[] byteArrayCache; @@ -92,10 +91,10 @@ public int getBytesPerRow() { @SuppressWarnings("unused") @DoNotStrip public byte[] toByteArray() { - switch (imageProxy.getFormat()) { + switch (image.getFormat()) { case ImageFormat.YUV_420_888: - ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); - ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer(); + ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); + ByteBuffer vuBuffer = image.getPlanes()[2].getBuffer(); int ySize = yBuffer.remaining(); int vuSize = vuBuffer.remaining(); @@ -108,13 +107,13 @@ public byte[] toByteArray() { return byteArrayCache; default: - throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!"); + throw new RuntimeException("Cannot convert Frame with Format " + image.getFormat() + " to byte array!"); } } @SuppressWarnings("unused") @DoNotStrip private void close() { - imageProxy.close(); + image.close(); } } diff --git a/android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt b/android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt index c80d71202d..d10d417bdd 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt @@ -15,6 +15,3 @@ val SizeF.bigger: Float val SizeF.smaller: Float get() = min(this.width, this.height) -fun areUltimatelyEqual(size1: Size, size2: Size): Boolean { - return size1.width * size1.height == size2.width * size2.height -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/AspectRatio.kt b/android/src/main/java/com/mrousavy/camera/utils/AspectRatio.kt deleted file mode 100644 index 8fb366aa6d..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/AspectRatio.kt +++ /dev/null @@ -1,28 +0,0 @@ -package com.mrousavy.camera.utils - -import androidx.camera.core.AspectRatio -import kotlin.math.abs -import kotlin.math.max -import kotlin.math.min - -private const val RATIO_4_3_VALUE = 4.0 / 3.0 -private const val RATIO_16_9_VALUE = 16.0 / 9.0 - -/** - * [androidx.camera.core.ImageAnalysisConfig] requires enum value of - * [androidx.camera.core.AspectRatio]. Currently it has values of 4:3 & 16:9. - * - * Detecting the most suitable ratio for dimensions provided in @params by counting absolute - * of preview ratio to one of the provided values. - * - * @param width - preview width - * @param height - preview height - * @return suitable aspect ratio - */ -fun aspectRatio(width: Int, height: Int): Int { - val previewRatio = max(width, height).toDouble() / min(width, height) - if (abs(previewRatio - RATIO_4_3_VALUE) <= abs(previewRatio - RATIO_16_9_VALUE)) { - return AspectRatio.RATIO_4_3 - } - return AspectRatio.RATIO_16_9 -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice.kt index 9bcd537d38..fbe822a728 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice.kt @@ -1,16 +1,12 @@ package com.mrousavy.camera.utils import android.hardware.camera2.CameraCharacteristics -import android.hardware.camera2.CameraExtensionCharacteristics import android.hardware.camera2.CameraManager import android.hardware.camera2.CameraMetadata import android.hardware.camera2.params.DynamicRangeProfiles import android.os.Build import android.util.Range import android.util.Size -import androidx.camera.core.CameraSelector -import androidx.camera.extensions.ExtensionMode -import androidx.camera.extensions.ExtensionsManager import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap @@ -21,8 +17,7 @@ import com.mrousavy.camera.parsers.parseVideoStabilizationMode import kotlin.math.PI import kotlin.math.atan -class CameraDevice(private val cameraManager: CameraManager, extensionsManager: ExtensionsManager, private val cameraId: String) { - private val cameraSelector = CameraSelector.Builder().byID(cameraId).build() +class CameraDevice(private val cameraManager: CameraManager, private val cameraId: String) { private val characteristics = cameraManager.getCameraCharacteristics(cameraId) private val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) ?: CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY private val capabilities = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES) ?: IntArray(0) @@ -32,7 +27,7 @@ class CameraDevice(private val cameraManager: CameraManager, extensionsManager: private val isMultiCam = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA) private val supportsDepthCapture = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT) private val supportsRawCapture = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_RAW) - private val supportsLowLightBoost = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.NIGHT) || extensions.contains(CameraExtensionCharacteristics.EXTENSION_NIGHT) + private val supportsLowLightBoost = false // TODO: supportsLowLightBoost private val lensFacing = characteristics.get(CameraCharacteristics.LENS_FACING)!! private val hasFlash = characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE) ?: false private val focalLengths = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) ?: FloatArray(0) @@ -50,7 +45,7 @@ class CameraDevice(private val cameraManager: CameraManager, extensionsManager: private val isoRange = characteristics.get(CameraCharacteristics.SENSOR_INFO_SENSITIVITY_RANGE) ?: Range(0, 0) private val digitalStabilizationModes = characteristics.get(CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES) ?: IntArray(0) private val opticalStabilizationModes = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_OPTICAL_STABILIZATION) ?: IntArray(0) - private val supportsPhotoHdr = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.HDR) || extensions.contains(CameraExtensionCharacteristics.EXTENSION_HDR) + private val supportsPhotoHdr = false // TODO: supportsPhotoHdr private val supportsVideoHdr = getHasVideoHdr() // see https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraSelector+byID.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraSelector+byID.kt deleted file mode 100644 index 4bc2a0c2ac..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraSelector+byID.kt +++ /dev/null @@ -1,25 +0,0 @@ -package com.mrousavy.camera.utils - -import android.annotation.SuppressLint -import androidx.camera.camera2.interop.Camera2CameraInfo -import androidx.camera.core.CameraSelector -import java.lang.IllegalArgumentException - -/** - * Create a new [CameraSelector] which selects the camera with the given [cameraId] - */ -@SuppressLint("UnsafeOptInUsageError") -fun CameraSelector.Builder.byID(cameraId: String): CameraSelector.Builder { - return this.addCameraFilter { cameras -> - cameras.filter { cameraInfoX -> - try { - val cameraInfo = Camera2CameraInfo.from(cameraInfoX) - return@filter cameraInfo.cameraId == cameraId - } catch (e: IllegalArgumentException) { - // Occurs when the [cameraInfoX] is not castable to a Camera2 Info object. - // We can ignore this error because the [getAvailableCameraDevices()] func only returns Camera2 devices. - return@filter false - } - } - } -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/DeviceFormat.kt b/android/src/main/java/com/mrousavy/camera/utils/DeviceFormat.kt deleted file mode 100644 index 3364d8c57b..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/DeviceFormat.kt +++ /dev/null @@ -1,33 +0,0 @@ -package com.mrousavy.camera.utils - -import android.util.Range -import android.util.Size -import com.facebook.react.bridge.ReadableMap - -class DeviceFormat(map: ReadableMap) { - val frameRateRanges: List> - val photoSize: Size - val videoSize: Size - - init { - frameRateRanges = map.getArray("frameRateRanges")!!.toArrayList().map { range -> - if (range is HashMap<*, *>) - rangeFactory(range["minFrameRate"], range["maxFrameRate"]) - else - throw IllegalArgumentException("DeviceFormat: frameRateRanges contained a Range that was not of type HashMap<*,*>! Actual Type: ${range?.javaClass?.name}") - } - photoSize = Size(map.getInt("photoWidth"), map.getInt("photoHeight")) - videoSize = Size(map.getInt("videoWidth"), map.getInt("videoHeight")) - } -} - -fun rangeFactory(minFrameRate: Any?, maxFrameRate: Any?): Range { - return when (minFrameRate) { - is Int -> Range(minFrameRate, maxFrameRate as Int) - is Double -> Range(minFrameRate.toInt(), (maxFrameRate as Double).toInt()) - else -> throw IllegalArgumentException( - "DeviceFormat: frameRateRanges contained a Range that didn't have minFrameRate/maxFrameRate of types Int/Double! " + - "Actual Type: ${minFrameRate?.javaClass?.name} & ${maxFrameRate?.javaClass?.name}" - ) - } -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/ExifInterface+buildMetadataMap.kt b/android/src/main/java/com/mrousavy/camera/utils/ExifInterface+buildMetadataMap.kt deleted file mode 100644 index e4a8c45a98..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/ExifInterface+buildMetadataMap.kt +++ /dev/null @@ -1,62 +0,0 @@ -package com.mrousavy.camera.utils - -import androidx.exifinterface.media.ExifInterface -import com.facebook.react.bridge.Arguments -import com.facebook.react.bridge.WritableMap - -fun ExifInterface.buildMetadataMap(): WritableMap { - val metadataMap = Arguments.createMap() - metadataMap.putInt("Orientation", this.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)) - - val tiffMap = Arguments.createMap() - tiffMap.putInt("ResolutionUnit", this.getAttributeInt(ExifInterface.TAG_RESOLUTION_UNIT, 0)) - tiffMap.putString("Software", this.getAttribute(ExifInterface.TAG_SOFTWARE)) - tiffMap.putString("Make", this.getAttribute(ExifInterface.TAG_MAKE)) - tiffMap.putString("DateTime", this.getAttribute(ExifInterface.TAG_DATETIME)) - tiffMap.putDouble("XResolution", this.getAttributeDouble(ExifInterface.TAG_X_RESOLUTION, 0.0)) - tiffMap.putString("Model", this.getAttribute(ExifInterface.TAG_MODEL)) - tiffMap.putDouble("YResolution", this.getAttributeDouble(ExifInterface.TAG_Y_RESOLUTION, 0.0)) - metadataMap.putMap("{TIFF}", tiffMap) - - val exifMap = Arguments.createMap() - exifMap.putString("DateTimeOriginal", this.getAttribute(ExifInterface.TAG_DATETIME_ORIGINAL)) - exifMap.putDouble("ExposureTime", this.getAttributeDouble(ExifInterface.TAG_EXPOSURE_TIME, 0.0)) - exifMap.putDouble("FNumber", this.getAttributeDouble(ExifInterface.TAG_F_NUMBER, 0.0)) - val lensSpecificationArray = Arguments.createArray() - this.getAttributeRange(ExifInterface.TAG_LENS_SPECIFICATION)?.forEach { lensSpecificationArray.pushInt(it.toInt()) } - exifMap.putArray("LensSpecification", lensSpecificationArray) - exifMap.putDouble("ExposureBiasValue", this.getAttributeDouble(ExifInterface.TAG_EXPOSURE_BIAS_VALUE, 0.0)) - exifMap.putInt("ColorSpace", this.getAttributeInt(ExifInterface.TAG_COLOR_SPACE, ExifInterface.COLOR_SPACE_S_RGB)) - exifMap.putInt("FocalLenIn35mmFilm", this.getAttributeInt(ExifInterface.TAG_FOCAL_LENGTH_IN_35MM_FILM, 0)) - exifMap.putDouble("BrightnessValue", this.getAttributeDouble(ExifInterface.TAG_BRIGHTNESS_VALUE, 0.0)) - exifMap.putInt("ExposureMode", this.getAttributeInt(ExifInterface.TAG_EXPOSURE_MODE, ExifInterface.EXPOSURE_MODE_AUTO.toInt())) - exifMap.putString("LensModel", this.getAttribute(ExifInterface.TAG_LENS_MODEL)) - exifMap.putInt("SceneType", this.getAttributeInt(ExifInterface.TAG_SCENE_TYPE, ExifInterface.SCENE_TYPE_DIRECTLY_PHOTOGRAPHED.toInt())) - exifMap.putInt("PixelXDimension", this.getAttributeInt(ExifInterface.TAG_PIXEL_X_DIMENSION, 0)) - exifMap.putDouble("ShutterSpeedValue", this.getAttributeDouble(ExifInterface.TAG_SHUTTER_SPEED_VALUE, 0.0)) - exifMap.putInt("SensingMethod", this.getAttributeInt(ExifInterface.TAG_SENSING_METHOD, ExifInterface.SENSOR_TYPE_NOT_DEFINED.toInt())) - val subjectAreaArray = Arguments.createArray() - this.getAttributeRange(ExifInterface.TAG_SUBJECT_AREA)?.forEach { subjectAreaArray.pushInt(it.toInt()) } - exifMap.putArray("SubjectArea", subjectAreaArray) - exifMap.putDouble("ApertureValue", this.getAttributeDouble(ExifInterface.TAG_APERTURE_VALUE, 0.0)) - exifMap.putString("SubsecTimeDigitized", this.getAttribute(ExifInterface.TAG_SUBSEC_TIME_DIGITIZED)) - exifMap.putDouble("FocalLength", this.getAttributeDouble(ExifInterface.TAG_FOCAL_LENGTH, 0.0)) - exifMap.putString("LensMake", this.getAttribute(ExifInterface.TAG_LENS_MAKE)) - exifMap.putString("SubsecTimeOriginal", this.getAttribute(ExifInterface.TAG_SUBSEC_TIME_ORIGINAL)) - exifMap.putString("OffsetTimeDigitized", this.getAttribute(ExifInterface.TAG_OFFSET_TIME_DIGITIZED)) - exifMap.putInt("PixelYDimension", this.getAttributeInt(ExifInterface.TAG_PIXEL_Y_DIMENSION, 0)) - val isoSpeedRatingsArray = Arguments.createArray() - this.getAttributeRange(ExifInterface.TAG_PHOTOGRAPHIC_SENSITIVITY)?.forEach { isoSpeedRatingsArray.pushInt(it.toInt()) } - exifMap.putArray("ISOSpeedRatings", isoSpeedRatingsArray) - exifMap.putInt("WhiteBalance", this.getAttributeInt(ExifInterface.TAG_WHITE_BALANCE, 0)) - exifMap.putString("DateTimeDigitized", this.getAttribute(ExifInterface.TAG_DATETIME_DIGITIZED)) - exifMap.putString("OffsetTimeOriginal", this.getAttribute(ExifInterface.TAG_OFFSET_TIME_ORIGINAL)) - exifMap.putString("ExifVersion", this.getAttribute(ExifInterface.TAG_EXIF_VERSION)) - exifMap.putString("OffsetTime", this.getAttribute(ExifInterface.TAG_OFFSET_TIME)) - exifMap.putInt("Flash", this.getAttributeInt(ExifInterface.TAG_FLASH, ExifInterface.FLAG_FLASH_FIRED.toInt())) - exifMap.putInt("ExposureProgram", this.getAttributeInt(ExifInterface.TAG_EXPOSURE_PROGRAM, ExifInterface.EXPOSURE_PROGRAM_NOT_DEFINED.toInt())) - exifMap.putInt("MeteringMode", this.getAttributeInt(ExifInterface.TAG_METERING_MODE, ExifInterface.METERING_MODE_UNKNOWN.toInt())) - metadataMap.putMap("{Exif}", exifMap) - - return metadataMap -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/ImageCapture+suspendables.kt b/android/src/main/java/com/mrousavy/camera/utils/ImageCapture+suspendables.kt deleted file mode 100644 index 5bbe16bdaa..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/ImageCapture+suspendables.kt +++ /dev/null @@ -1,41 +0,0 @@ -package com.mrousavy.camera.utils - -import androidx.camera.core.ImageCapture -import androidx.camera.core.ImageCaptureException -import androidx.camera.core.ImageProxy -import java.util.concurrent.Executor -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlin.coroutines.suspendCoroutine - -suspend inline fun ImageCapture.takePicture(options: ImageCapture.OutputFileOptions, executor: Executor) = suspendCoroutine { cont -> - this.takePicture( - options, executor, - object : ImageCapture.OnImageSavedCallback { - override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { - cont.resume(outputFileResults) - } - - override fun onError(exception: ImageCaptureException) { - cont.resumeWithException(exception) - } - } - ) -} - -suspend inline fun ImageCapture.takePicture(executor: Executor) = suspendCoroutine { cont -> - this.takePicture( - executor, - object : ImageCapture.OnImageCapturedCallback() { - override fun onCaptureSuccess(image: ImageProxy) { - super.onCaptureSuccess(image) - cont.resume(image) - } - - override fun onError(exception: ImageCaptureException) { - super.onError(exception) - cont.resumeWithException(exception) - } - } - ) -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/ImageProxy+isRaw.kt b/android/src/main/java/com/mrousavy/camera/utils/ImageProxy+isRaw.kt deleted file mode 100644 index b86e6c6da7..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/ImageProxy+isRaw.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.mrousavy.camera.utils - -import android.graphics.ImageFormat -import androidx.camera.core.ImageProxy - -val ImageProxy.isRaw: Boolean - get() { - return when (format) { - ImageFormat.RAW_SENSOR, ImageFormat.RAW10, ImageFormat.RAW12, ImageFormat.RAW_PRIVATE -> true - else -> false - } - } diff --git a/android/src/main/java/com/mrousavy/camera/utils/ImageProxy+save.kt b/android/src/main/java/com/mrousavy/camera/utils/ImageProxy+save.kt deleted file mode 100644 index 73deec4c88..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/ImageProxy+save.kt +++ /dev/null @@ -1,127 +0,0 @@ -package com.mrousavy.camera.utils - -import android.graphics.Bitmap -import android.graphics.BitmapFactory -import android.graphics.ImageFormat -import android.graphics.Matrix -import android.util.Log -import androidx.camera.core.ImageProxy -import androidx.exifinterface.media.ExifInterface -import com.mrousavy.camera.CameraView -import com.mrousavy.camera.InvalidFormatError -import java.io.ByteArrayOutputStream -import java.io.File -import java.io.FileOutputStream -import java.nio.ByteBuffer -import kotlin.system.measureTimeMillis - -// TODO: Fix this flip() function (this outputs a black image) -fun flip(imageBytes: ByteArray, imageWidth: Int): ByteArray { - // separate out the sub arrays - var holder = ByteArray(imageBytes.size) - var subArray = ByteArray(imageWidth) - var subCount = 0 - for (i in imageBytes.indices) { - subArray[subCount] = imageBytes[i] - subCount++ - if (i % imageWidth == 0) { - subArray.reverse() - if (i == imageWidth) { - holder = subArray - } else { - holder += subArray - } - subCount = 0 - subArray = ByteArray(imageWidth) - } - } - subArray = ByteArray(imageWidth) - System.arraycopy(imageBytes, imageBytes.size - imageWidth, subArray, 0, subArray.size) - return holder + subArray -} - -// TODO: This function is slow. Figure out a faster way to flip images, preferably via directly manipulating the byte[] Exif flags -fun flipImage(imageBytes: ByteArray): ByteArray { - val bitmap = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size) - val matrix = Matrix() - - val exif = ExifInterface(imageBytes.inputStream()) - val orientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_UNDEFINED) - - when (orientation) { - ExifInterface.ORIENTATION_ROTATE_180 -> { - matrix.setRotate(180f) - matrix.postScale(-1f, 1f) - } - ExifInterface.ORIENTATION_FLIP_VERTICAL -> { - matrix.setRotate(180f) - } - ExifInterface.ORIENTATION_TRANSPOSE -> { - matrix.setRotate(90f) - } - ExifInterface.ORIENTATION_ROTATE_90 -> { - matrix.setRotate(90f) - matrix.postScale(-1f, 1f) - } - ExifInterface.ORIENTATION_TRANSVERSE -> { - matrix.setRotate(-90f) - } - ExifInterface.ORIENTATION_ROTATE_270 -> { - matrix.setRotate(-90f) - matrix.postScale(-1f, 1f) - } - } - - val newBitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) - val stream = ByteArrayOutputStream() - newBitmap.compress(Bitmap.CompressFormat.JPEG, 100, stream) - return stream.toByteArray() -} - -fun ImageProxy.save(file: File, flipHorizontally: Boolean) { - when (format) { - // TODO: ImageFormat.RAW_SENSOR - // TODO: ImageFormat.DEPTH_JPEG - ImageFormat.JPEG -> { - val buffer = planes[0].buffer - var bytes = ByteArray(buffer.remaining()) - - // copy image from buffer to byte array - buffer.get(bytes) - - if (flipHorizontally) { - val milliseconds = measureTimeMillis { - bytes = flipImage(bytes) - } - Log.i(CameraView.TAG_PERF, "Flipping Image took $milliseconds ms.") - } - - val output = FileOutputStream(file) - output.write(bytes) - output.close() - } - ImageFormat.YUV_420_888 -> { - // "prebuffer" simply contains the meta information about the following planes. - val prebuffer = ByteBuffer.allocate(16) - prebuffer.putInt(width) - .putInt(height) - .putInt(planes[1].pixelStride) - .putInt(planes[1].rowStride) - - val output = FileOutputStream(file) - output.write(prebuffer.array()) // write meta information to file - // Now write the actual planes. - var buffer: ByteBuffer - var bytes: ByteArray - - for (i in 0..2) { - buffer = planes[i].buffer - bytes = ByteArray(buffer.remaining()) // makes byte array large enough to hold image - buffer.get(bytes) // copies image from buffer to byte array - output.write(bytes) // write the byte array to file - } - output.close() - } - else -> throw InvalidFormatError(format) - } -} diff --git a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java index ff984a5cb9..376ea16464 100644 --- a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java +++ b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java @@ -1,9 +1,8 @@ package com.mrousavy.camera.example; +import android.media.Image; import android.util.Log; -import androidx.camera.core.ImageProxy; - import com.facebook.react.bridge.ReadableNativeMap; import com.facebook.react.bridge.WritableNativeArray; import com.facebook.react.bridge.WritableNativeMap; @@ -18,7 +17,7 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { @Override public Object callback(@NotNull Frame frame, @Nullable ReadableNativeMap params) { HashMap hashMap = params != null ? params.toHashMap() : new HashMap<>(); - ImageProxy image = frame.getImageProxy(); + Image image = frame.getImage(); Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + hashMap.size() + " parameters:"); diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 4990e62aff..8e717b5131 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -710,7 +710,7 @@ SPEC CHECKSUMS: RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8 RNVectorIcons: 8b5bb0fa61d54cd2020af4f24a51841ce365c7e9 SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17 - VisionCamera: d0112c5121c8fc785ed9c2a1e4a557ae22088709 + VisionCamera: 2ee7d7545925a09d996c4bd70438ebc64714eccc Yoga: 8796b55dba14d7004f980b54bcc9833ee45b28ce PODFILE CHECKSUM: ab9c06b18c63e741c04349c0fd630c6d3145081c From 1485910d4fbc6e3c280fed9773f055b71915cee8 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 1 Aug 2023 12:20:59 +0200 Subject: [PATCH 002/180] fix: Run View Finder on UI Thread --- .../camera/frameprocessor/VisionCameraProxy.kt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index 246a745794..86e8382a1d 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -2,10 +2,12 @@ package com.mrousavy.camera.frameprocessor import android.util.Log import androidx.annotation.Keep +import androidx.annotation.UiThread import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.ReactApplicationContext import com.facebook.react.bridge.ReadableNativeMap +import com.facebook.react.bridge.UiThreadUtil import com.facebook.react.turbomodule.core.CallInvokerHolderImpl import com.facebook.react.uimanager.UIManagerHelper import com.mrousavy.camera.CameraView @@ -41,6 +43,7 @@ class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler) } + @UiThread private fun findCameraViewById(viewId: Int): CameraView { Log.d(TAG, "Finding view $viewId...") val ctx = mContext.get() @@ -52,15 +55,19 @@ class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: @DoNotStrip @Keep fun setFrameProcessor(viewId: Int, frameProcessor: FrameProcessor) { - val view = findCameraViewById(viewId) - view.frameProcessor = frameProcessor + UiThreadUtil.runOnUiThread { + val view = findCameraViewById(viewId) + view.frameProcessor = frameProcessor + } } @DoNotStrip @Keep fun removeFrameProcessor(viewId: Int) { - val view = findCameraViewById(viewId) - view.frameProcessor = null + UiThreadUtil.runOnUiThread { + val view = findCameraViewById(viewId) + view.frameProcessor = null + } } @DoNotStrip From bdb61a0e5d6e3f75e33af1f266911a6392d5478a Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 1 Aug 2023 16:37:17 +0200 Subject: [PATCH 003/180] Open Camera, set up Threads --- .../java/com/mrousavy/camera/CameraQueues.kt | 36 +++++++ .../java/com/mrousavy/camera/CameraView.kt | 95 +++++++++++++------ .../com/mrousavy/camera/CameraViewManager.kt | 3 +- .../com/mrousavy/camera/CameraViewModule.kt | 4 +- .../main/java/com/mrousavy/camera/Errors.kt | 2 + .../frameprocessor/VisionCameraProxy.kt | 4 +- .../frameprocessor/VisionCameraScheduler.java | 9 +- .../CameraDevice+createCaptureSession.kt | 71 ++++++++++++++ .../camera/parsers/CameraError+String.kt | 14 +++ src/CameraError.ts | 2 + 10 files changed, 201 insertions(+), 39 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/CameraQueues.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt new file mode 100644 index 0000000000..ea90e8cb3f --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt @@ -0,0 +1,36 @@ +package com.mrousavy.camera + +import android.os.Handler +import android.os.HandlerThread +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.asCoroutineDispatcher +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import kotlin.coroutines.CoroutineContext + +class CameraQueues { + companion object { + val cameraQueue = CameraQueue("mrousavy/VisionCamera.main") + val videoQueue = CameraQueue("mrousavy/VisionCamera.video") + } + + class CameraQueue(name: String) { + val executor: ExecutorService + val handler: Handler + val coroutineScope: CoroutineScope + private val thread: HandlerThread + + init { + thread = HandlerThread(name) + thread.start() + handler = Handler(thread.looper) + executor = Executors.newSingleThreadExecutor() + coroutineScope = CoroutineScope(executor.asCoroutineDispatcher()) + } + + protected fun finalize() { + thread.quitSafely() + } + } +} + diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index d26c84a458..a16e1392fe 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -5,18 +5,33 @@ import android.annotation.SuppressLint import android.content.Context import android.content.pm.PackageManager import android.content.res.Configuration +import android.graphics.ImageFormat import android.hardware.camera2.* +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.params.OutputConfiguration +import android.hardware.camera2.params.SessionConfiguration +import android.media.ImageReader +import android.media.ImageReader.OnImageAvailableListener +import android.os.Build +import android.os.Handler +import android.os.HandlerThread import android.util.Log import android.view.* import android.widget.FrameLayout import androidx.core.content.ContextCompat import androidx.lifecycle.* import com.facebook.react.bridge.* +import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.parsers.SessionType +import com.mrousavy.camera.parsers.SurfaceOutput +import com.mrousavy.camera.parsers.createCaptureSession +import com.mrousavy.camera.parsers.parseCameraError import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import java.lang.IllegalArgumentException import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors import kotlin.math.max import kotlin.math.min @@ -49,8 +64,8 @@ import kotlin.math.min // TODO: takePhoto() return with jsi::Value Image reference for faster capture @Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that. -@SuppressLint("ClickableViewAccessibility", "ViewConstructor") -class CameraView(context: Context, private val frameProcessorThread: ExecutorService) : FrameLayout(context) { +@SuppressLint("ClickableViewAccessibility", "ViewConstructor", "MissingPermission") +class CameraView(context: Context) : FrameLayout(context) { companion object { const val TAG = "CameraView" const val TAG_PERF = "CameraView.performance" @@ -84,6 +99,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer // private properties private var isMounted = false + private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager public var frameProcessor: FrameProcessor? = null @@ -169,35 +185,58 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer * Configures the camera capture session. This should only be called when the camera device changes. */ private fun configureSession() { - try { - val startTime = System.currentTimeMillis() - Log.i(TAG, "Configuring session...") - if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { - throw CameraPermissionError() + val startTime = System.currentTimeMillis() + Log.i(TAG, "Configuring session...") + if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { + throw CameraPermissionError() + } + val cameraId = cameraId ?: throw NoCameraDeviceError() + + Log.i(TAG, "Opening Camera $cameraId...") + cameraManager.openCamera(cameraId, object: CameraDevice.StateCallback() { + override fun onOpened(camera: CameraDevice) { + Log.i(TAG, "Successfully opened Camera Device $cameraId!") + CameraQueues.cameraQueue.coroutineScope.launch { + configureCamera(camera) + } } - if (cameraId == null) { - throw NoCameraDeviceError() + + override fun onDisconnected(camera: CameraDevice) { + Log.i(TAG, "Camera Device $cameraId has been disconnected! Waiting for reconnect to continue session..") + invokeOnError(CameraDisconnectedError(cameraId)) } - // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f - // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f - - val duration = System.currentTimeMillis() - startTime - Log.i(TAG_PERF, "Session configured in $duration ms!") - invokeOnInitialized() - } catch (exc: Throwable) { - Log.e(TAG, "Failed to configure session: ${exc.message}") - throw when (exc) { - is CameraError -> exc - is IllegalArgumentException -> { - if (exc.message?.contains("too many use cases") == true) { - ParallelVideoProcessingNotSupportedError(exc) - } else { - InvalidCameraDeviceError(exc) - } - } - else -> UnknownCameraError(exc) + override fun onError(camera: CameraDevice, error: Int) { + Log.e(TAG, "Failed to open Camera Device $cameraId! Error: $error (${parseCameraError(error)})") + invokeOnError(CameraCannotBeOpenedError(cameraId, parseCameraError(error))) } - } + }, null) + + // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f + // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f + } + + private suspend fun configureCamera(camera: CameraDevice) { + val imageReader = ImageReader.newInstance(1920, 1080, ImageFormat.YUV_420_888, 2) + + imageReader.setOnImageAvailableListener({ reader -> + Log.d(TAG, "New Image available!") + val image = reader.acquireLatestImage() + // TODO: Rotation + // TODO: isMirrored + val frame = Frame(image, System.currentTimeMillis(), Surface.ROTATION_0, false) + frameProcessor?.call(frame) + }, null) + + val frameProcessorOutput = SurfaceOutput(imageReader.surface) + val outputs = listOf(frameProcessorOutput) + val session = camera.createCaptureSession(SessionType.REGULAR, outputs, CameraQueues.cameraQueue) + + val captureRequest = camera.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) + captureRequest.addTarget(imageReader.surface) + session.setRepeatingRequest(captureRequest.build(), null, null) + + Log.i(TAG, "Successfully configured Camera Session!") + invokeOnInitialized() } } diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt index 383ee66930..9ebf665b87 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt @@ -11,8 +11,7 @@ import com.facebook.react.uimanager.annotations.ReactProp class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManager() { public override fun createViewInstance(context: ThemedReactContext): CameraView { - val cameraViewModule = context.getNativeModule(CameraViewModule::class.java)!! - return CameraView(context, cameraViewModule.frameProcessorThread) + return CameraView(context) } override fun onAfterUpdateTransaction(view: CameraView) { diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index 27d110d256..bdeb7f0594 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -30,12 +30,10 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ var RequestCode = 10 } - var frameProcessorThread: ExecutorService = Executors.newSingleThreadExecutor() private val coroutineScope = CoroutineScope(Dispatchers.Default) // TODO: or Dispatchers.Main? override fun invalidate() { super.invalidate() - frameProcessorThread.shutdown() if (coroutineScope.isActive) { coroutineScope.cancel("CameraViewModule has been destroyed.") } @@ -55,7 +53,7 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod(isBlockingSynchronousMethod = true) fun installFrameProcessorBindings(): Boolean { return try { - val proxy = VisionCameraProxy(reactApplicationContext, frameProcessorThread) + val proxy = VisionCameraProxy(reactApplicationContext) VisionCameraInstaller.install(proxy) true } catch (e: Error) { diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 2147e756b9..03df5288bc 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -54,6 +54,8 @@ class LowLightBoostNotContainedInFormatError : CameraError( ) class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") +class CameraCannotBeOpenedError(cameraId: String, error: String) : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") +class CameraDisconnectedError(cameraId: String) : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected!") class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.") diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index 86e8382a1d..a44c1fc209 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -17,7 +17,7 @@ import java.util.concurrent.ExecutorService @Suppress("KotlinJniMissingFunction") // we use fbjni. -class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { +class VisionCameraProxy(context: ReactApplicationContext) { companion object { const val TAG = "VisionCameraProxy" init { @@ -38,7 +38,7 @@ class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: init { val jsCallInvokerHolder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl val jsRuntimeHolder = context.javaScriptContextHolder.get() - mScheduler = VisionCameraScheduler(frameProcessorThread) + mScheduler = VisionCameraScheduler() mContext = WeakReference(context) mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler) } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java index cdc43a3944..456430675c 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java @@ -2,6 +2,8 @@ import com.facebook.jni.HybridData; import com.facebook.proguard.annotations.DoNotStrip; +import com.mrousavy.camera.CameraQueues; + import java.util.concurrent.ExecutorService; @SuppressWarnings("JavaJniMissingFunction") // using fbjni here @@ -9,10 +11,8 @@ public class VisionCameraScheduler { @SuppressWarnings({"unused", "FieldCanBeLocal"}) @DoNotStrip private final HybridData mHybridData; - private final ExecutorService frameProcessorThread; - public VisionCameraScheduler(ExecutorService frameProcessorThread) { - this.frameProcessorThread = frameProcessorThread; + public VisionCameraScheduler() { mHybridData = initHybrid(); } @@ -22,6 +22,7 @@ public VisionCameraScheduler(ExecutorService frameProcessorThread) { @SuppressWarnings("unused") @DoNotStrip private void scheduleTrigger() { - frameProcessorThread.submit(this::trigger); + CameraQueues.CameraQueue videoQueue = CameraQueues.Companion.getVideoQueue(); + videoQueue.getExecutor().submit(this::trigger); } } diff --git a/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt new file mode 100644 index 0000000000..6978396de8 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt @@ -0,0 +1,71 @@ +package com.mrousavy.camera.parsers + +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.params.OutputConfiguration +import android.hardware.camera2.params.SessionConfiguration +import android.os.Build +import android.view.Surface +import com.mrousavy.camera.CameraQueues +import java.util.concurrent.Executor +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine + +data class SurfaceOutput(val surface: Surface, + val isMirrored: Boolean = false, + val streamUseCase: Long = 0x0 /* DEFAULT */, + val dynamicRangeProfile: Long = 0 /* STANDARD */) + +enum class SessionType { + REGULAR, + HIGH_SPEED, + VENDOR; + + fun toSessionType(): Int { + // TODO: Use actual enum when we are on API Level 28 + return when(this) { + REGULAR -> 0 /* CameraDevice.SESSION_OPERATION_MODE_NORMAL */ + HIGH_SPEED -> 1 /* CameraDevice.SESSION_OPERATION_MODE_CONSTRAINED_HIGH_SPEED */ + VENDOR -> 0x8000 /* CameraDevice.SESSION_OPERATION_MODE_VENDOR_START */ + } + } +} + +suspend fun CameraDevice.createCaptureSession(sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { + return suspendCoroutine { continuation -> + + val callback = object : CameraCaptureSession.StateCallback() { + override fun onConfigured(session: CameraCaptureSession) { + continuation.resume(session) + } + + override fun onConfigureFailed(session: CameraCaptureSession) { + continuation.resumeWithException(RuntimeException("Failed to configure the Camera Session!")) + } + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + val outputConfigurations = outputs.map { + val result = OutputConfiguration(it.surface) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + result.mirrorMode = if (it.isMirrored) OutputConfiguration.MIRROR_MODE_H else OutputConfiguration.MIRROR_MODE_NONE + result.dynamicRangeProfile = it.dynamicRangeProfile + result.streamUseCase = it.streamUseCase + } + return@map result + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + // API >28 + val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) + this.createCaptureSession(config) + } else { + // API >24 + this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) + } + } else { + // API <23 + this.createCaptureSession(outputs.map { it.surface }, callback, queue.handler) + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt new file mode 100644 index 0000000000..3388206133 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt @@ -0,0 +1,14 @@ +package com.mrousavy.camera.parsers + +import android.hardware.camera2.CameraDevice + +fun parseCameraError(error: Int): String { + return when (error) { + CameraDevice.StateCallback.ERROR_CAMERA_IN_USE -> "camera-already-in-use" + CameraDevice.StateCallback.ERROR_MAX_CAMERAS_IN_USE -> "too-many-open-cameras" + CameraDevice.StateCallback.ERROR_CAMERA_DISABLED -> "camera-is-disabled-by-android" + CameraDevice.StateCallback.ERROR_CAMERA_DEVICE -> "unknown-camera-device-error" + CameraDevice.StateCallback.ERROR_CAMERA_SERVICE -> "unknown-fatal-camera-service-error" + else -> "unknown-error" + } +} diff --git a/src/CameraError.ts b/src/CameraError.ts index ae5cb2ca06..9865868b28 100644 --- a/src/CameraError.ts +++ b/src/CameraError.ts @@ -23,6 +23,8 @@ export type FormatError = | 'format/invalid-color-space'; export type SessionError = | 'session/camera-not-ready' + | 'session/camera-cannot-be-opened' + | 'session/camera-has-been-disconnected' | 'session/audio-session-setup-failed' | 'session/audio-in-use-by-other-app' | 'session/audio-session-failed-to-activate'; From be20f8be6c976631ed6a35d969fe83955a681130 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 1 Aug 2023 16:43:39 +0200 Subject: [PATCH 004/180] fix init --- .../src/main/java/com/mrousavy/camera/CameraView.kt | 7 +++++-- .../parsers/CameraDevice+createCaptureSession.kt | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index a16e1392fe..9de508e5bf 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -221,12 +221,15 @@ class CameraView(context: Context) : FrameLayout(context) { imageReader.setOnImageAvailableListener({ reader -> Log.d(TAG, "New Image available!") - val image = reader.acquireLatestImage() + val image = reader.acquireNextImage() + if (image == null) { + Log.e(TAG, "Failed to get new Image from ImageReader, dropping it...") + } // TODO: Rotation // TODO: isMirrored val frame = Frame(image, System.currentTimeMillis(), Surface.ROTATION_0, false) frameProcessor?.call(frame) - }, null) + }, CameraQueues.videoQueue.handler) val frameProcessorOutput = SurfaceOutput(imageReader.surface) val outputs = listOf(frameProcessorOutput) diff --git a/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt index 6978396de8..0a2e80100e 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt @@ -14,8 +14,8 @@ import kotlin.coroutines.suspendCoroutine data class SurfaceOutput(val surface: Surface, val isMirrored: Boolean = false, - val streamUseCase: Long = 0x0 /* DEFAULT */, - val dynamicRangeProfile: Long = 0 /* STANDARD */) + val streamUseCase: Long? = null, + val dynamicRangeProfile: Long? = null) enum class SessionType { REGULAR, @@ -49,9 +49,9 @@ suspend fun CameraDevice.createCaptureSession(sessionType: SessionType, outputs: val outputConfigurations = outputs.map { val result = OutputConfiguration(it.surface) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - result.mirrorMode = if (it.isMirrored) OutputConfiguration.MIRROR_MODE_H else OutputConfiguration.MIRROR_MODE_NONE - result.dynamicRangeProfile = it.dynamicRangeProfile - result.streamUseCase = it.streamUseCase + if (it.isMirrored) result.mirrorMode = OutputConfiguration.MIRROR_MODE_H + if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile + if (it.streamUseCase != null) result.streamUseCase = it.streamUseCase } return@map result } From 9099967a8d291188c6fff4c17f3fe3c02d040706 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 1 Aug 2023 16:51:58 +0200 Subject: [PATCH 005/180] Mirror if needed --- .../java/com/mrousavy/camera/CameraView.kt | 9 +++++---- ...CameraDevice.kt => CameraDeviceDetails.kt} | 20 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) rename android/src/main/java/com/mrousavy/camera/utils/{CameraDevice.kt => CameraDeviceDetails.kt} (91%) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 9de508e5bf..2d7e8fb5ee 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -219,19 +219,20 @@ class CameraView(context: Context) : FrameLayout(context) { private suspend fun configureCamera(camera: CameraDevice) { val imageReader = ImageReader.newInstance(1920, 1080, ImageFormat.YUV_420_888, 2) + val characteristics = cameraManager.getCameraCharacteristics(camera.id) + val isMirrored = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT + imageReader.setOnImageAvailableListener({ reader -> Log.d(TAG, "New Image available!") val image = reader.acquireNextImage() if (image == null) { Log.e(TAG, "Failed to get new Image from ImageReader, dropping it...") } - // TODO: Rotation - // TODO: isMirrored - val frame = Frame(image, System.currentTimeMillis(), Surface.ROTATION_0, false) + val frame = Frame(image, System.currentTimeMillis(), inputRotation, isMirrored) frameProcessor?.call(frame) }, CameraQueues.videoQueue.handler) - val frameProcessorOutput = SurfaceOutput(imageReader.surface) + val frameProcessorOutput = SurfaceOutput(imageReader.surface, isMirrored) val outputs = listOf(frameProcessorOutput) val session = camera.createCaptureSession(SessionType.REGULAR, outputs, CameraQueues.cameraQueue) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt similarity index 91% rename from android/src/main/java/com/mrousavy/camera/utils/CameraDevice.kt rename to android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index fbe822a728..8d1dab901b 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -1,6 +1,8 @@ package com.mrousavy.camera.utils import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraExtensionCharacteristics import android.hardware.camera2.CameraManager import android.hardware.camera2.CameraMetadata import android.hardware.camera2.params.DynamicRangeProfiles @@ -17,23 +19,23 @@ import com.mrousavy.camera.parsers.parseVideoStabilizationMode import kotlin.math.PI import kotlin.math.atan -class CameraDevice(private val cameraManager: CameraManager, private val cameraId: String) { - private val characteristics = cameraManager.getCameraCharacteristics(cameraId) +class CameraDeviceDetails(private val cameraManager: CameraManager, private val cameraDevice: CameraDevice) { + private val characteristics = cameraManager.getCameraCharacteristics(cameraDevice.id) private val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) ?: CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY private val capabilities = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES) ?: IntArray(0) private val extensions = getSupportedExtensions() // device characteristics - private val isMultiCam = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA) - private val supportsDepthCapture = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT) + private val isMultiCam = capabilities.contains(11 /* TODO: CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA */) + private val supportsDepthCapture = capabilities.contains(8 /* TODO: CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT */) private val supportsRawCapture = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_RAW) - private val supportsLowLightBoost = false // TODO: supportsLowLightBoost + private val supportsLowLightBoost = extensions.contains(4 /* TODO: CameraExtensionCharacteristics.EXTENSION_NIGHT */) private val lensFacing = characteristics.get(CameraCharacteristics.LENS_FACING)!! private val hasFlash = characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE) ?: false private val focalLengths = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) ?: FloatArray(0) private val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! private val name = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) - else null) ?: "${parseLensFacing(lensFacing)} (${cameraId})" + else null) ?: "${parseLensFacing(lensFacing)} (${cameraDevice.id})" // "formats" (all possible configurations for this device) private val zoomRange = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) characteristics.get(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE) @@ -45,7 +47,7 @@ class CameraDevice(private val cameraManager: CameraManager, private val cameraI private val isoRange = characteristics.get(CameraCharacteristics.SENSOR_INFO_SENSITIVITY_RANGE) ?: Range(0, 0) private val digitalStabilizationModes = characteristics.get(CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES) ?: IntArray(0) private val opticalStabilizationModes = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_OPTICAL_STABILIZATION) ?: IntArray(0) - private val supportsPhotoHdr = false // TODO: supportsPhotoHdr + private val supportsPhotoHdr = extensions.contains(3 /* TODO: CameraExtensionCharacteristics.EXTENSION_HDR */) private val supportsVideoHdr = getHasVideoHdr() // see https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture @@ -54,7 +56,7 @@ class CameraDevice(private val cameraManager: CameraManager, private val cameraI // get extensions (HDR, Night Mode, ..) private fun getSupportedExtensions(): List { return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - val extensions = cameraManager.getCameraExtensionCharacteristics(cameraId) + val extensions = cameraManager.getCameraExtensionCharacteristics(cameraDevice.id) extensions.supportedExtensions } else { emptyList() @@ -191,7 +193,7 @@ class CameraDevice(private val cameraManager: CameraManager, private val cameraI // convert to React Native JS object (map) fun toMap(): ReadableMap { val map = Arguments.createMap() - map.putString("id", cameraId) + map.putString("id", cameraDevice.id) map.putArray("devices", getDeviceTypes()) map.putString("position", parseLensFacing(lensFacing)) map.putString("name", name) From 96a7cfdf8f5e7a40d683e476109aafb560e18c63 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 1 Aug 2023 17:02:36 +0200 Subject: [PATCH 006/180] Try PreviewView --- .../java/com/mrousavy/camera/CameraView.kt | 35 +++++++++++++++---- .../com/mrousavy/camera/CameraViewModule.kt | 2 +- .../camera/utils/CameraDeviceDetails.kt | 10 +++--- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 2d7e8fb5ee..4eea054f06 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -68,7 +68,6 @@ import kotlin.math.min class CameraView(context: Context) : FrameLayout(context) { companion object { const val TAG = "CameraView" - const val TAG_PERF = "CameraView.performance" private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "fps", "hdr", "lowLightBoost", "photo", "video", "enableFrameProcessor") private val arrayListOfZoom = arrayListOf("zoom") @@ -101,6 +100,10 @@ class CameraView(context: Context) : FrameLayout(context) { private var isMounted = false private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + // session + private var cameraSession: CameraCaptureSession? = null + private val previewView = SurfaceView(context) + public var frameProcessor: FrameProcessor? = null private val inputRotation: Int @@ -128,6 +131,23 @@ class CameraView(context: Context) : FrameLayout(context) { private var maxZoom: Float = 1f init { + this.installHierarchyFitter() + previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) + previewView.holder.addCallback(object : SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) { + Log.i(TAG, "PreviewView Surface created!") + if (cameraId != null) configureSession() + } + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { + Log.i(TAG, "PreviewView Surface resized!") + } + + override fun surfaceDestroyed(holder: SurfaceHolder) { + Log.i(TAG, "PreviewView Surface destroyed!") + } + }) + addView(previewView) } override fun onConfigurationChanged(newConfig: Configuration?) { @@ -163,7 +183,7 @@ class CameraView(context: Context) : FrameLayout(context) { // TODO: updateLifecycleState() } if (shouldReconfigureSession) { - configureSession() + // configureSession() } if (shouldReconfigureZoom) { val zoomClamped = max(min(zoom, maxZoom), minZoom) @@ -185,7 +205,6 @@ class CameraView(context: Context) : FrameLayout(context) { * Configures the camera capture session. This should only be called when the camera device changes. */ private fun configureSession() { - val startTime = System.currentTimeMillis() Log.i(TAG, "Configuring session...") if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { throw CameraPermissionError() @@ -222,6 +241,7 @@ class CameraView(context: Context) : FrameLayout(context) { val characteristics = cameraManager.getCameraCharacteristics(camera.id) val isMirrored = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT + // Setting up Video / Frame Processor imageReader.setOnImageAvailableListener({ reader -> Log.d(TAG, "New Image available!") val image = reader.acquireNextImage() @@ -233,12 +253,15 @@ class CameraView(context: Context) : FrameLayout(context) { }, CameraQueues.videoQueue.handler) val frameProcessorOutput = SurfaceOutput(imageReader.surface, isMirrored) - val outputs = listOf(frameProcessorOutput) - val session = camera.createCaptureSession(SessionType.REGULAR, outputs, CameraQueues.cameraQueue) + val previewOutput = SurfaceOutput(previewView.holder.surface, isMirrored) + val outputs = listOf(frameProcessorOutput, previewOutput) + cameraSession = camera.createCaptureSession(SessionType.REGULAR, outputs, CameraQueues.cameraQueue) + // Start Video / Frame Processor val captureRequest = camera.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) captureRequest.addTarget(imageReader.surface) - session.setRepeatingRequest(captureRequest.build(), null, null) + captureRequest.addTarget(previewView.holder.surface) + cameraSession!!.setRepeatingRequest(captureRequest.build(), null, null) Log.i(TAG, "Successfully configured Camera Session!") invokeOnInitialized() diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index bdeb7f0594..3eebed6b32 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -146,7 +146,7 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ val devices = Arguments.createArray() manager.cameraIdList.forEach { cameraId -> - val device = CameraDevice(manager, cameraId) + val device = CameraDeviceDetails(manager, cameraId) devices.pushMap(device.toMap()) } promise.resolve(devices) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 8d1dab901b..f293301e01 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -19,8 +19,8 @@ import com.mrousavy.camera.parsers.parseVideoStabilizationMode import kotlin.math.PI import kotlin.math.atan -class CameraDeviceDetails(private val cameraManager: CameraManager, private val cameraDevice: CameraDevice) { - private val characteristics = cameraManager.getCameraCharacteristics(cameraDevice.id) +class CameraDeviceDetails(private val cameraManager: CameraManager, private val cameraId: String) { + private val characteristics = cameraManager.getCameraCharacteristics(cameraId) private val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) ?: CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY private val capabilities = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES) ?: IntArray(0) private val extensions = getSupportedExtensions() @@ -35,7 +35,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private val focalLengths = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) ?: FloatArray(0) private val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! private val name = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) - else null) ?: "${parseLensFacing(lensFacing)} (${cameraDevice.id})" + else null) ?: "${parseLensFacing(lensFacing)} (${cameraId})" // "formats" (all possible configurations for this device) private val zoomRange = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) characteristics.get(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE) @@ -56,7 +56,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val // get extensions (HDR, Night Mode, ..) private fun getSupportedExtensions(): List { return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - val extensions = cameraManager.getCameraExtensionCharacteristics(cameraDevice.id) + val extensions = cameraManager.getCameraExtensionCharacteristics(cameraId) extensions.supportedExtensions } else { emptyList() @@ -193,7 +193,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val // convert to React Native JS object (map) fun toMap(): ReadableMap { val map = Arguments.createMap() - map.putString("id", cameraDevice.id) + map.putString("id", cameraId) map.putArray("devices", getDeviceTypes()) map.putString("position", parseLensFacing(lensFacing)) map.putString("name", name) From bd518dbfdf18bce49318c01adc78cec92e17864c Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 1 Aug 2023 21:39:42 +0200 Subject: [PATCH 007/180] Use max resolution --- .../mrousavy/camera/CameraView+RecordVideo.kt | 12 ++ .../java/com/mrousavy/camera/CameraView.kt | 107 ++++++++++++++---- .../CameraDevice+createCaptureSession.kt | 37 ++++-- 3 files changed, 127 insertions(+), 29 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt index eb27ef51c6..d4310ff65a 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt @@ -3,8 +3,11 @@ package com.mrousavy.camera import android.Manifest import android.annotation.SuppressLint import android.content.pm.PackageManager +import android.media.Image +import android.util.Log import androidx.core.content.ContextCompat import com.facebook.react.bridge.* +import com.mrousavy.camera.frameprocessor.Frame import java.io.File import java.text.SimpleDateFormat import java.util.* @@ -44,3 +47,12 @@ fun CameraView.stopRecording() { // TODO: stopRecording() // TODO: disable torch again } + +fun CameraView.onFrame(frame: Frame) { + Log.d(CameraView.TAG, "New Frame available!") + if (frameProcessor != null) { + frameProcessor?.call(frame) + } + + // TODO: Record Video here using MediaCodec +} diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 4eea054f06..98db3b6ce3 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -10,8 +10,10 @@ import android.hardware.camera2.* import android.hardware.camera2.CameraDevice import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration +import android.hardware.camera2.params.StreamConfigurationMap import android.media.ImageReader import android.media.ImageReader.OnImageAvailableListener +import android.media.MediaRecorder import android.os.Build import android.os.Handler import android.os.HandlerThread @@ -23,10 +25,12 @@ import androidx.lifecycle.* import com.facebook.react.bridge.* import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.parsers.OutputType import com.mrousavy.camera.parsers.SessionType import com.mrousavy.camera.parsers.SurfaceOutput import com.mrousavy.camera.parsers.createCaptureSession import com.mrousavy.camera.parsers.parseCameraError +import com.mrousavy.camera.parsers.parseImageFormat import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import java.lang.IllegalArgumentException @@ -90,6 +94,7 @@ class CameraView(context: Context) : FrameLayout(context) { var hdr: Boolean? = null // nullable bool var colorSpace: String? = null var lowLightBoost: Boolean? = null // nullable bool + var previewType: String = "native" // other props var isActive = false var torch = "off" @@ -103,6 +108,7 @@ class CameraView(context: Context) : FrameLayout(context) { // session private var cameraSession: CameraCaptureSession? = null private val previewView = SurfaceView(context) + private var isPreviewSurfaceReady = false public var frameProcessor: FrameProcessor? = null @@ -136,6 +142,7 @@ class CameraView(context: Context) : FrameLayout(context) { previewView.holder.addCallback(object : SurfaceHolder.Callback { override fun surfaceCreated(holder: SurfaceHolder) { Log.i(TAG, "PreviewView Surface created!") + isPreviewSurfaceReady = true if (cameraId != null) configureSession() } @@ -145,6 +152,7 @@ class CameraView(context: Context) : FrameLayout(context) { override fun surfaceDestroyed(holder: SurfaceHolder) { Log.i(TAG, "PreviewView Surface destroyed!") + isPreviewSurfaceReady = false } }) addView(previewView) @@ -230,40 +238,99 @@ class CameraView(context: Context) : FrameLayout(context) { invokeOnError(CameraCannotBeOpenedError(cameraId, parseCameraError(error))) } }, null) - - // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f - // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f } private suspend fun configureCamera(camera: CameraDevice) { - val imageReader = ImageReader.newInstance(1920, 1080, ImageFormat.YUV_420_888, 2) + if (cameraSession != null) { + // Close any existing Session + cameraSession?.close() + } val characteristics = cameraManager.getCameraCharacteristics(camera.id) val isMirrored = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - // Setting up Video / Frame Processor - imageReader.setOnImageAvailableListener({ reader -> - Log.d(TAG, "New Image available!") - val image = reader.acquireNextImage() - if (image == null) { - Log.e(TAG, "Failed to get new Image from ImageReader, dropping it...") - } - val frame = Frame(image, System.currentTimeMillis(), inputRotation, isMirrored) - frameProcessor?.call(frame) - }, CameraQueues.videoQueue.handler) + // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f + // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f + + val outputs = arrayListOf() + + if (photo == true) { + // Photo output: High quality still images + val format = ImageFormat.JPEG + // TODO: Let user configure photoSize with format (or new builder API) + val photoSize = config.getOutputSizes(format).maxBy { it.height * it.width } + val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, format, 1) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() + Log.d(TAG, "Photo captured! ${image.width} x ${image.height}") + image.close() + }, CameraQueues.cameraQueue.handler) + + Log.i(TAG, "Creating ${photoSize.width}x${photoSize.height} photo output. (Format: $format)") + val photoOutput = SurfaceOutput(imageReader.surface, isMirrored, OutputType.PHOTO) + outputs.add(photoOutput) + } + + if (video == true || enableFrameProcessor) { + // Video or Frame Processor output: High resolution repeating images + val format = getVideoFormat(config) + // TODO: Let user configure videoSize with format (or new builder API) + val videoSize = config.getOutputSizes(format).maxBy { it.height * it.width } + val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, format, 2) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() + if (image == null) { + Log.w(TAG, "Failed to get new Image from ImageReader, dropping a Frame...") + return@setOnImageAvailableListener + } + val frame = Frame(image, System.currentTimeMillis(), inputRotation, isMirrored) + onFrame(frame) + }, CameraQueues.videoQueue.handler) + + Log.i(TAG, "Creating ${videoSize.width}x${videoSize.height} video output. (Format: $format)") + val videoOutput = SurfaceOutput(imageReader.surface, isMirrored, OutputType.VIDEO) + outputs.add(videoOutput) + } + + if (previewType == "native") { + // Preview output: Low resolution repeating images + val previewOutput = SurfaceOutput(previewView.holder.surface, isMirrored, OutputType.PREVIEW) + outputs.add(previewOutput) + } - val frameProcessorOutput = SurfaceOutput(imageReader.surface, isMirrored) - val previewOutput = SurfaceOutput(previewView.holder.surface, isMirrored) - val outputs = listOf(frameProcessorOutput, previewOutput) cameraSession = camera.createCaptureSession(SessionType.REGULAR, outputs, CameraQueues.cameraQueue) - // Start Video / Frame Processor + // Start all repeating requests (Video, Frame Processor, Preview) val captureRequest = camera.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) - captureRequest.addTarget(imageReader.surface) - captureRequest.addTarget(previewView.holder.surface) + outputs.forEach { output -> + if (output.isRepeating) captureRequest.addTarget(output.surface) + } cameraSession!!.setRepeatingRequest(captureRequest.build(), null, null) Log.i(TAG, "Successfully configured Camera Session!") invokeOnInitialized() } + + private fun getVideoFormat(config: StreamConfigurationMap): Int { + val formats = config.outputFormats + if (formats.contains(ImageFormat.YUV_420_888)) { + return ImageFormat.YUV_420_888 + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + if (formats.contains(ImageFormat.YUV_422_888)) { + return ImageFormat.YUV_422_888 + } + if (formats.contains(ImageFormat.YUV_444_888)) { + return ImageFormat.YUV_444_888 + } + if (formats.contains(ImageFormat.FLEX_RGB_888)) { + return ImageFormat.FLEX_RGB_888 + } + if (formats.contains(ImageFormat.FLEX_RGBA_8888)) { + return ImageFormat.FLEX_RGBA_8888 + } + } + return formats[0] + } } diff --git a/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt index 0a2e80100e..b0f3abf2b3 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt @@ -2,6 +2,7 @@ package com.mrousavy.camera.parsers import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraMetadata import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration import android.os.Build @@ -12,26 +13,44 @@ import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine -data class SurfaceOutput(val surface: Surface, - val isMirrored: Boolean = false, - val streamUseCase: Long? = null, - val dynamicRangeProfile: Long? = null) - enum class SessionType { REGULAR, - HIGH_SPEED, - VENDOR; + HIGH_SPEED; fun toSessionType(): Int { // TODO: Use actual enum when we are on API Level 28 return when(this) { REGULAR -> 0 /* CameraDevice.SESSION_OPERATION_MODE_NORMAL */ HIGH_SPEED -> 1 /* CameraDevice.SESSION_OPERATION_MODE_CONSTRAINED_HIGH_SPEED */ - VENDOR -> 0x8000 /* CameraDevice.SESSION_OPERATION_MODE_VENDOR_START */ } } } +enum class OutputType { + PHOTO, + VIDEO, + PREVIEW, + VIDEO_AND_PREVIEW; + + fun toOutputType(): Long { + // TODO: Use actual enum when we are on API Level 28 + return when(this) { + PHOTO -> 0x2 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_STILL_CAPTURE */ + VIDEO -> 0x3 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_VIDEO_RECORD */ + PREVIEW -> 0x1 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW */ + VIDEO_AND_PREVIEW -> 0x4 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW_VIDEO_STILL */ + } + } +} + +data class SurfaceOutput(val surface: Surface, + val isMirrored: Boolean = false, + val outputType: OutputType? = null, + val dynamicRangeProfile: Long? = null) { + val isRepeating: Boolean + get() = outputType == OutputType.VIDEO || outputType == OutputType.PREVIEW || outputType == OutputType.VIDEO_AND_PREVIEW +} + suspend fun CameraDevice.createCaptureSession(sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { return suspendCoroutine { continuation -> @@ -51,7 +70,7 @@ suspend fun CameraDevice.createCaptureSession(sessionType: SessionType, outputs: if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { if (it.isMirrored) result.mirrorMode = OutputConfiguration.MIRROR_MODE_H if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile - if (it.streamUseCase != null) result.streamUseCase = it.streamUseCase + if (it.outputType != null) result.streamUseCase = it.outputType.toOutputType() } return@map result } From 0c5e04cf5464619821e251db65c3bd8139305cc0 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 11:45:15 +0200 Subject: [PATCH 008/180] Add `hardwareLevel` property --- .../java/com/mrousavy/camera/CameraView.kt | 39 +++++-------------- .../camera/parsers/HardwareLevel+String.kt | 14 +++++++ .../CameraDevice+createCaptureSession.kt | 15 +++++-- .../camera/utils/CameraDeviceDetails.kt | 2 + ios/CameraViewManager.swift | 1 + src/CameraDevice.ts | 6 +++ 6 files changed, 44 insertions(+), 33 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt rename android/src/main/java/com/mrousavy/camera/{parsers => utils}/CameraDevice+createCaptureSession.kt (81%) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 98db3b6ce3..9d2217bccc 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -6,17 +6,11 @@ import android.content.Context import android.content.pm.PackageManager import android.content.res.Configuration import android.graphics.ImageFormat +import android.graphics.PixelFormat import android.hardware.camera2.* import android.hardware.camera2.CameraDevice -import android.hardware.camera2.params.OutputConfiguration -import android.hardware.camera2.params.SessionConfiguration import android.hardware.camera2.params.StreamConfigurationMap import android.media.ImageReader -import android.media.ImageReader.OnImageAvailableListener -import android.media.MediaRecorder -import android.os.Build -import android.os.Handler -import android.os.HandlerThread import android.util.Log import android.view.* import android.widget.FrameLayout @@ -25,17 +19,13 @@ import androidx.lifecycle.* import com.facebook.react.bridge.* import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor -import com.mrousavy.camera.parsers.OutputType -import com.mrousavy.camera.parsers.SessionType -import com.mrousavy.camera.parsers.SurfaceOutput -import com.mrousavy.camera.parsers.createCaptureSession +import com.mrousavy.camera.utils.OutputType +import com.mrousavy.camera.utils.SessionType +import com.mrousavy.camera.utils.SurfaceOutput +import com.mrousavy.camera.utils.createCaptureSession import com.mrousavy.camera.parsers.parseCameraError -import com.mrousavy.camera.parsers.parseImageFormat import com.mrousavy.camera.utils.* import kotlinx.coroutines.* -import java.lang.IllegalArgumentException -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors import kotlin.math.max import kotlin.math.min @@ -299,7 +289,7 @@ class CameraView(context: Context) : FrameLayout(context) { outputs.add(previewOutput) } - cameraSession = camera.createCaptureSession(SessionType.REGULAR, outputs, CameraQueues.cameraQueue) + cameraSession = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, CameraQueues.cameraQueue) // Start all repeating requests (Video, Frame Processor, Preview) val captureRequest = camera.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) @@ -317,20 +307,11 @@ class CameraView(context: Context) : FrameLayout(context) { if (formats.contains(ImageFormat.YUV_420_888)) { return ImageFormat.YUV_420_888 } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - if (formats.contains(ImageFormat.YUV_422_888)) { - return ImageFormat.YUV_422_888 - } - if (formats.contains(ImageFormat.YUV_444_888)) { - return ImageFormat.YUV_444_888 - } - if (formats.contains(ImageFormat.FLEX_RGB_888)) { - return ImageFormat.FLEX_RGB_888 - } - if (formats.contains(ImageFormat.FLEX_RGBA_8888)) { - return ImageFormat.FLEX_RGBA_8888 - } + if (formats.contains(PixelFormat.RGB_888)) { + return PixelFormat.RGB_888; } + Log.w(TAG, "Couldn't find YUV_420_888 or RGB_888 format for Video " + + "Recording, using unknown format instead.. (${formats[0]})") return formats[0] } } diff --git a/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt new file mode 100644 index 0000000000..adc4947f00 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt @@ -0,0 +1,14 @@ +package com.mrousavy.camera.parsers + +import android.hardware.camera2.CameraCharacteristics + +fun parseHardwareLevel(hardwareLevel: Int): String { + return when (hardwareLevel) { + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY -> "legacy" + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED -> "limited" + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_EXTERNAL -> "limited" + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL -> "full" + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3 -> "full" + else -> "legacy" + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt similarity index 81% rename from android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt rename to android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index b0f3abf2b3..3271fa16dd 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -1,14 +1,17 @@ -package com.mrousavy.camera.parsers +package com.mrousavy.camera.utils import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice -import android.hardware.camera2.CameraMetadata +import android.hardware.camera2.CameraManager import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration import android.os.Build +import android.util.Log import android.view.Surface import com.mrousavy.camera.CameraQueues -import java.util.concurrent.Executor +import com.mrousavy.camera.CameraView +import com.mrousavy.camera.parsers.parseHardwareLevel import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine @@ -51,7 +54,7 @@ data class SurfaceOutput(val surface: Surface, get() = outputType == OutputType.VIDEO || outputType == OutputType.PREVIEW || outputType == OutputType.VIDEO_AND_PREVIEW } -suspend fun CameraDevice.createCaptureSession(sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { +suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { return suspendCoroutine { continuation -> val callback = object : CameraCaptureSession.StateCallback() { @@ -64,6 +67,10 @@ suspend fun CameraDevice.createCaptureSession(sessionType: SessionType, outputs: } } + val characteristics = cameraManager.getCameraCharacteristics(this.id) + val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! + Log.i(CameraView.TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { val outputConfigurations = outputs.map { val result = OutputConfiguration(it.surface) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index f293301e01..903b3b4597 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -13,6 +13,7 @@ import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import com.mrousavy.camera.parsers.bigger +import com.mrousavy.camera.parsers.parseHardwareLevel import com.mrousavy.camera.parsers.parseImageFormat import com.mrousavy.camera.parsers.parseLensFacing import com.mrousavy.camera.parsers.parseVideoStabilizationMode @@ -208,6 +209,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val map.putDouble("minZoom", minZoom) map.putDouble("maxZoom", maxZoom) map.putDouble("neutralZoom", 1.0) // Zoom is always relative to 1.0 on Android + map.putString("hardwareLevel", parseHardwareLevel(hardwareLevel)) map.putArray("formats", getFormats()) diff --git a/ios/CameraViewManager.swift b/ios/CameraViewManager.swift index b35b798169..472b209495 100644 --- a/ios/CameraViewManager.swift +++ b/ios/CameraViewManager.swift @@ -122,6 +122,7 @@ final class CameraViewManager: RCTViewManager { "supportsRawCapture": false, // TODO: supportsRawCapture "supportsLowLightBoost": $0.isLowLightBoostSupported, "supportsFocus": $0.isFocusPointOfInterestSupported, + "hardwareLevel": "full", "formats": $0.formats.map { format -> [String: Any] in format.toDictionary() }, diff --git a/src/CameraDevice.ts b/src/CameraDevice.ts index b52ae7b77d..e3a1ca0a53 100644 --- a/src/CameraDevice.ts +++ b/src/CameraDevice.ts @@ -281,4 +281,10 @@ export interface CameraDevice { * Specifies whether this device supports focusing ({@linkcode Camera.focus | Camera.focus(...)}) */ supportsFocus: boolean; + /** + * The hardware level of the Camera. + * - On Android, some older devices are running at a `legacy` or `limited` level which means they are running in a backwards compatible mode. + * - On iOS, all devices are `full`. + */ + hardwareLevel: 'legacy' | 'limited' | 'full'; } From b43ba63e9acfde53f3ff9c0a73289dbbca2ffbb8 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 12:25:59 +0200 Subject: [PATCH 009/180] Check if output type is supported --- .../CameraDevice+createCaptureSession.kt | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 3271fa16dd..5df64354aa 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -54,6 +54,23 @@ data class SurfaceOutput(val surface: Surface, get() = outputType == OutputType.VIDEO || outputType == OutputType.PREVIEW || outputType == OutputType.VIDEO_AND_PREVIEW } +fun supportsOutputType(characteristics: CameraCharacteristics, outputType: OutputType): Boolean { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val availableUseCases = characteristics.get(CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES) + if (availableUseCases != null) { + if (availableUseCases.contains(outputType.toOutputType())) { + return true + } + } + } + val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! + if (hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3 || hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL) { + return true + } + + return false +} + suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { return suspendCoroutine { continuation -> @@ -77,7 +94,9 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sess if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { if (it.isMirrored) result.mirrorMode = OutputConfiguration.MIRROR_MODE_H if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile - if (it.outputType != null) result.streamUseCase = it.outputType.toOutputType() + if (it.outputType != null && supportsOutputType(characteristics, it.outputType)) { + result.streamUseCase = it.outputType.toOutputType() + } } return@map result } From a2a294c00708e711a3c39247fa2c29e881d98c44 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 15:04:23 +0200 Subject: [PATCH 010/180] Replace `frameRateRanges` with `minFps` and `maxFps` --- android/src/main/cpp/FrameHostObject.cpp | 5 +++ android/src/main/cpp/java-bindings/JFrame.cpp | 5 +++ android/src/main/cpp/java-bindings/JFrame.h | 1 + .../java/com/mrousavy/camera/CameraView.kt | 3 ++ .../main/java/com/mrousavy/camera/Errors.kt | 4 +- .../mrousavy/camera/frameprocessor/Frame.java | 9 ++++ .../camera/parsers/ImageFormat+String.kt | 7 +++- .../camera/utils/CameraDeviceDetails.kt | 41 +++++-------------- docs/docs/guides/FORMATS.mdx | 9 +--- example/src/CameraPage.tsx | 18 ++++---- ios/CameraError.swift | 4 +- .../AVCaptureDevice.Format+isBetterThan.swift | 7 +--- ...AVCaptureDevice.Format+matchesFilter.swift | 19 ++++----- .../AVCaptureDevice.Format+toDictionary.swift | 20 ++++++--- ios/Frame Processor/FrameHostObject.mm | 14 +++++++ src/CameraDevice.ts | 13 +++--- src/Frame.ts | 5 +++ src/PixelFormat.ts | 8 ++-- src/utils/FormatFilter.ts | 16 +------- 19 files changed, 106 insertions(+), 102 deletions(-) diff --git a/android/src/main/cpp/FrameHostObject.cpp b/android/src/main/cpp/FrameHostObject.cpp index b20ccac17f..b0ca2d5f1f 100644 --- a/android/src/main/cpp/FrameHostObject.cpp +++ b/android/src/main/cpp/FrameHostObject.cpp @@ -37,6 +37,7 @@ std::vector FrameHostObject::getPropertyNames(jsi::Runtime& rt) result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation"))); result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored"))); result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp"))); + result.push_back(jsi::PropNameID::forUtf8(rt, std::string("pixelFormat"))); // Conversion result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString"))); result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer"))); @@ -136,6 +137,10 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr auto string = this->frame->getOrientation(); return jsi::String::createFromUtf8(runtime, string->toStdString()); } + if (name == "pixelFormat") { + auto string = this->frame->getPixelFormat(); + return jsi::String::createFromUtf8(runtime, string->toStdString()); + } if (name == "timestamp") { return jsi::Value(static_cast(this->frame->getTimestamp())); } diff --git a/android/src/main/cpp/java-bindings/JFrame.cpp b/android/src/main/cpp/java-bindings/JFrame.cpp index d79658dc97..dce59e0a08 100644 --- a/android/src/main/cpp/java-bindings/JFrame.cpp +++ b/android/src/main/cpp/java-bindings/JFrame.cpp @@ -42,6 +42,11 @@ local_ref JFrame::getOrientation() const { return getOrientationMethod(self()); } +local_ref JFrame::getPixelFormat() const { + static const auto getPixelFormatMethod = getClass()->getMethod("getPixelFormat"); + return getPixelFormatMethod(self()); +} + int JFrame::getPlanesCount() const { static const auto getPlanesCountMethod = getClass()->getMethod("getPlanesCount"); return getPlanesCountMethod(self()); diff --git a/android/src/main/cpp/java-bindings/JFrame.h b/android/src/main/cpp/java-bindings/JFrame.h index 8d9949bcc0..ac6e43c94b 100644 --- a/android/src/main/cpp/java-bindings/JFrame.h +++ b/android/src/main/cpp/java-bindings/JFrame.h @@ -24,6 +24,7 @@ struct JFrame : public JavaClass { int getBytesPerRow() const; jlong getTimestamp() const; local_ref getOrientation() const; + local_ref getPixelFormat() const; local_ref toByteArray() const; void close(); }; diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 9d2217bccc..2b0bc45794 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -11,6 +11,7 @@ import android.hardware.camera2.* import android.hardware.camera2.CameraDevice import android.hardware.camera2.params.StreamConfigurationMap import android.media.ImageReader +import android.os.Build import android.util.Log import android.view.* import android.widget.FrameLayout @@ -265,6 +266,7 @@ class CameraView(context: Context) : FrameLayout(context) { if (video == true || enableFrameProcessor) { // Video or Frame Processor output: High resolution repeating images val format = getVideoFormat(config) + config.inputFormats // TODO: Let user configure videoSize with format (or new builder API) val videoSize = config.getOutputSizes(format).maxBy { it.height * it.width } val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, format, 2) @@ -304,6 +306,7 @@ class CameraView(context: Context) : FrameLayout(context) { private fun getVideoFormat(config: StreamConfigurationMap): Int { val formats = config.outputFormats + Log.i(TAG, "Device supports ${formats.size} output formats: ${formats.joinToString(", ")}") if (formats.contains(ImageFormat.YUV_420_888)) { return ImageFormat.YUV_420_888 } diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 03df5288bc..27a5246019 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -41,11 +41,11 @@ class ParallelVideoProcessingNotSupportedError(cause: Throwable) : CameraError(" "video processing (`video={true}` + `frameProcessor={...}`). Disable either `video` or `frameProcessor`. To find out if a device supports parallel video processing, check the `supportsParallelVideoProcessing` property on the CameraDevice. " + "See https://react-native-vision-camera.com/docs/guides/devices#the-supportsparallelvideoprocessing-prop for more information.", cause) -class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given FPS were not valid for the currently selected format. Make sure you select a format which `frameRateRanges` includes $fps FPS!") +class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given format cannot run at $fps FPS! Make sure your FPS is lower than `format.maxFps` but higher than `format.minFps`.") class HdrNotContainedInFormatError : CameraError( "format", "invalid-hdr", "The currently selected format does not support HDR capture! " + - "Make sure you select a format which `frameRateRanges` includes `supportsPhotoHDR`!" + "Make sure you select a format which includes `supportsPhotoHDR`!" ) class LowLightBoostNotContainedInFormatError : CameraError( "format", "invalid-low-light-boost", diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index a09ca53ebf..e8eb945917 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -1,6 +1,9 @@ package com.mrousavy.camera.frameprocessor; +import static com.mrousavy.camera.parsers.ImageFormat_StringKt.parseImageFormat; + import android.graphics.ImageFormat; +import android.graphics.PixelFormat; import android.media.Image; import com.facebook.proguard.annotations.DoNotStrip; import java.nio.ByteBuffer; @@ -74,6 +77,12 @@ public String getOrientation() { return "portrait"; } + @SuppressWarnings("unused") + @DoNotStrip + public String getPixelFormat() { + return parseImageFormat(image.getFormat()); + } + @SuppressWarnings("unused") @DoNotStrip public int getPlanesCount() { diff --git a/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt index b37b346528..7d85829d37 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt @@ -1,6 +1,7 @@ package com.mrousavy.camera.parsers import android.graphics.ImageFormat +import android.graphics.PixelFormat /** * Parses ImageFormat/PixelFormat int to a string representation useable for the TypeScript types. @@ -8,6 +9,10 @@ import android.graphics.ImageFormat fun parseImageFormat(imageFormat: Int): String { return when (imageFormat) { ImageFormat.YUV_420_888 -> "yuv" + ImageFormat.JPEG -> "rgb" + PixelFormat.RGB_888 -> "rgb" + else -> "unknown" + /* ImageFormat.YUV_422_888 -> "yuv" ImageFormat.YUV_444_888 -> "yuv" ImageFormat.JPEG -> "jpeg" @@ -17,8 +22,6 @@ fun parseImageFormat(imageFormat: Int): String { ImageFormat.HEIC -> "heic" ImageFormat.PRIVATE -> "private" ImageFormat.DEPTH16 -> "depth-16" - else -> "unknown" - /* ImageFormat.UNKNOWN -> "TODOFILL" ImageFormat.RGB_565 -> "TODOFILL" ImageFormat.YV12 -> "TODOFILL" diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 903b3b4597..5dd72b5547 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -75,21 +75,6 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val return false } - private fun createFrameRateRanges(ranges: Array>): ReadableArray { - val array = Arguments.createArray() - ranges.forEach { range -> - val map = Arguments.createMap() - map.putInt("minFrameRate", range.lower) - map.putInt("maxFrameRate", range.upper) - array.pushMap(map) - } - return array - } - - private fun createFrameRateRanges(minFps: Int, maxFps: Int): ReadableArray { - return createFrameRateRanges(arrayOf(Range(minFps, maxFps))) - } - private fun createColorSpaces(): ReadableArray { val array = Arguments.createArray() array.pushString("yuv") @@ -139,7 +124,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val return 2 * atan(sensorSize.bigger / (focalLengths[0] * 2)) * (180 / PI) } - private fun buildFormatMap(outputSize: Size, outputFormat: Int, fpsRanges: ReadableArray): ReadableMap { + private fun buildFormatMap(outputSize: Size, outputFormat: Int, fpsRange: Range): ReadableMap { val highResSizes = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) cameraConfig.getHighResolutionOutputSizes(outputFormat) else null) ?: emptyArray() val map = Arguments.createMap() @@ -157,37 +142,33 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val map.putString("autoFocusSystem", "contrast-detection") // TODO: Is this wrong? map.putArray("videoStabilizationModes", createStabilizationModes()) map.putString("pixelFormat", parseImageFormat(outputFormat)) - map.putArray("frameRateRanges", fpsRanges) + map.putInt("minFps", fpsRange.lower) + map.putInt("maxFps", fpsRange.upper) return map } private fun getFormats(): ReadableArray { val array = Arguments.createArray() - val highSpeedSizes = cameraConfig.highSpeedVideoSizes - val outputFormats = cameraConfig.outputFormats outputFormats.forEach { outputFormat -> // Normal Video/Photo Sizes - val outputSizes = cameraConfig.getOutputSizes(outputFormat) + val outputSizes = cameraConfig.getOutputSizes(outputFormat).toMutableList() + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + // High resolution Photo sizes that are not able to run at 20FPS+ + outputSizes.addAll(cameraConfig.getHighResolutionOutputSizes(outputFormat)) + } outputSizes.forEach { outputSize -> val frameDuration = cameraConfig.getOutputMinFrameDuration(outputFormat, outputSize) val maxFps = (1.0 / (frameDuration.toDouble() / 1000000000)).toInt() - val minFps = 1 - val map = buildFormatMap(outputSize, outputFormat, createFrameRateRanges(minFps, maxFps)) - array.pushMap(map) - } - - // High-Speed (Slow Motion) Video Sizes - highSpeedSizes.forEach { outputSize -> - val highSpeedRanges = cameraConfig.getHighSpeedVideoFpsRangesFor(outputSize) - - val map = buildFormatMap(outputSize, outputFormat, createFrameRateRanges(highSpeedRanges)) + val map = buildFormatMap(outputSize, outputFormat, Range(1, maxFps)) array.pushMap(map) } } + // TODO: Add high-speed video ranges (high-fps / slow-motion) + return array } diff --git a/docs/docs/guides/FORMATS.mdx b/docs/docs/guides/FORMATS.mdx index 4f63024104..60d9bdce80 100644 --- a/docs/docs/guides/FORMATS.mdx +++ b/docs/docs/guides/FORMATS.mdx @@ -39,13 +39,6 @@ You can also manually get all camera devices and decide which device to use base This example shows how you would pick the format with the _highest frame rate_: ```tsx -function getMaxFps(format: CameraDeviceFormat): number { - return format.frameRateRanges.reduce((prev, curr) => { - if (curr.maxFrameRate > prev) return curr.maxFrameRate - else return prev - }, 0) -} - function App() { const devices = useCameraDevices('wide-angle-camera') const device = devices.back @@ -53,7 +46,7 @@ function App() { const format = useMemo(() => { return device?.formats.reduce((prev, curr) => { if (prev == null) return curr - if (getMaxFps(curr) > getMaxFps(prev)) return curr + if (curr.maxFps > prev.maxFps) return curr else return prev }, undefined) }, [device?.formats]) diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index 29cbbb2a0b..73c5d6edd9 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -11,7 +11,7 @@ import { useFrameProcessor, VideoFile, } from 'react-native-vision-camera'; -import { Camera, frameRateIncluded } from 'react-native-vision-camera'; +import { Camera } from 'react-native-vision-camera'; import { CONTENT_SPACING, MAX_ZOOM_FACTOR, SAFE_AREA_PADDING } from './Constants'; import Reanimated, { Extrapolate, interpolate, useAnimatedGestureHandler, useAnimatedProps, useSharedValue } from 'react-native-reanimated'; import { useEffect } from 'react'; @@ -72,13 +72,13 @@ export function CameraPage({ navigation }: Props): React.ReactElement { return 30; } - const supportsHdrAt60Fps = formats.some((f) => f.supportsVideoHDR && f.frameRateRanges.some((r) => frameRateIncluded(r, 60))); + const supportsHdrAt60Fps = formats.some((f) => f.supportsVideoHDR && f.maxFps >= 60); if (enableHdr && !supportsHdrAt60Fps) { // User has enabled HDR, but HDR is not supported at 60 FPS. return 30; } - const supports60Fps = formats.some((f) => f.frameRateRanges.some((r) => frameRateIncluded(r, 60))); + const supports60Fps = formats.some((f) => f.maxFps >= 60); if (!supports60Fps) { // 60 FPS is not supported by any format. return 30; @@ -90,7 +90,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { const supportsCameraFlipping = useMemo(() => devices.back != null && devices.front != null, [devices.back, devices.front]); const supportsFlash = device?.hasFlash ?? false; const supportsHdr = useMemo(() => formats.some((f) => f.supportsVideoHDR || f.supportsPhotoHDR), [formats]); - const supports60Fps = useMemo(() => formats.some((f) => f.frameRateRanges.some((rate) => frameRateIncluded(rate, 60))), [formats]); + const supports60Fps = useMemo(() => formats.some((f) => f.maxFps >= 60), [formats]); const canToggleNightMode = enableNightMode ? true // it's enabled so you have to be able to turn it off again : (device?.supportsLowLightBoost ?? false) || fps > 30; // either we have native support, or we can lower the FPS @@ -105,7 +105,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { } // find the first format that includes the given FPS - return result.find((f) => f.frameRateRanges.some((r) => frameRateIncluded(r, fps))); + return result.find((f) => f.maxFps >= fps); }, [formats, fps, enableHdr]); //#region Animated Zoom @@ -221,9 +221,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { const frameProcessor = useFrameProcessor((frame) => { 'worklet'; - console.log(`Width: ${frame.width}`); - const result = examplePlugin(frame); - console.log('Example Plugin: ', result); + console.log(frame.timestamp, frame.toString(), frame.pixelFormat); }, []); return ( @@ -245,9 +243,11 @@ export function CameraPage({ navigation }: Props): React.ReactElement { onError={onError} enableZoomGesture={false} animatedProps={cameraAnimatedProps} - audio={hasMicrophonePermission} enableFpsGraph={true} orientation="portrait" + photo={true} + video={true} + audio={hasMicrophonePermission} frameProcessor={frameProcessor} /> diff --git a/ios/CameraError.swift b/ios/CameraError.swift index cb8678a282..46ff64b5fc 100644 --- a/ios/CameraError.swift +++ b/ios/CameraError.swift @@ -132,9 +132,9 @@ enum FormatError { case .invalidFormat: return "The given format was invalid. Did you check if the current device supports the given format by using `getAvailableCameraDevices(...)`?" case let .invalidFps(fps): - return "The given FPS were not valid for the currently selected format. Make sure you select a format which `frameRateRanges` includes \(fps) FPS!" + return "The given format cannot run at \(fps) FPS! Make sure your FPS is lower than `format.maxFps` but higher than `format.minFps`." case .invalidHdr: - return "The currently selected format does not support HDR capture! Make sure you select a format which `frameRateRanges` includes `supportsPhotoHDR`!" + return "The currently selected format does not support HDR capture! Make sure you select a format which includes `supportsPhotoHDR`!" case let .invalidColorSpace(colorSpace): return "The currently selected format does not support the colorSpace \"\(colorSpace)\"! " + "Make sure you select a format which `colorSpaces` includes \"\(colorSpace)\"!" diff --git a/ios/Extensions/AVCaptureDevice.Format+isBetterThan.swift b/ios/Extensions/AVCaptureDevice.Format+isBetterThan.swift index 1156526231..76ec2cc3df 100644 --- a/ios/Extensions/AVCaptureDevice.Format+isBetterThan.swift +++ b/ios/Extensions/AVCaptureDevice.Format+isBetterThan.swift @@ -30,11 +30,8 @@ extension AVCaptureDevice.Format { } // compare max fps - if let leftMaxFps = videoSupportedFrameRateRanges.max(by: { $0.maxFrameRate > $1.maxFrameRate }), - let rightMaxFps = other.videoSupportedFrameRateRanges.max(by: { $0.maxFrameRate > $1.maxFrameRate }) { - if leftMaxFps.maxFrameRate > rightMaxFps.maxFrameRate { - return true - } + if maxFrameRate > other.maxFrameRate { + return true } return false diff --git a/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift b/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift index 35789a6c99..fa17f03dfc 100644 --- a/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift +++ b/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift @@ -50,7 +50,7 @@ extension AVCaptureDevice.Format { } } if let maxZoom = filter.value(forKey: "maxZoom") as? NSNumber { - if videoMaxZoomFactor != CGFloat(maxZoom.floatValue) { + if videoMaxZoomFactor != CGFloat(maxZoom.doubleValue) { return false } } @@ -61,18 +61,13 @@ extension AVCaptureDevice.Format { return false } } - if let frameRateRanges = filter.value(forKey: "frameRateRanges") as? [NSDictionary] { - let allFrameRateRangesIncluded = videoSupportedFrameRateRanges.allSatisfy { range -> Bool in - frameRateRanges.contains { dict -> Bool in - guard let max = dict.value(forKey: "maxFrameRate") as? NSNumber, - let min = dict.value(forKey: "minFrameRate") as? NSNumber - else { - return false - } - return range.maxFrameRate == max.doubleValue && range.minFrameRate == min.doubleValue - } + if let minFps = filter.value(forKey: "minFps") as? NSNumber { + if minFrameRate != Float64(minFps.doubleValue) { + return false } - if !allFrameRateRangesIncluded { + } + if let maxFps = filter.value(forKey: "maxFps") as? NSNumber { + if maxFrameRate != Float64(maxFps.doubleValue) { return false } } diff --git a/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift b/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift index fec83cae3b..b728ccf47f 100644 --- a/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift +++ b/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift @@ -20,6 +20,18 @@ extension AVCaptureDevice.Format { var videoStabilizationModes: [AVCaptureVideoStabilizationMode] { return getAllVideoStabilizationModes().filter { self.isVideoStabilizationModeSupported($0) } } + var minFrameRate: Float64 { + let maxRange = videoSupportedFrameRateRanges.max { l, r in + return l.maxFrameRate < r.maxFrameRate + } + return maxRange?.maxFrameRate ?? 0 + } + var maxFrameRate: Float64 { + let maxRange = videoSupportedFrameRateRanges.max { l, r in + return l.maxFrameRate < r.maxFrameRate + } + return maxRange?.maxFrameRate ?? 0 + } func toDictionary() -> [String: Any] { var dict: [String: Any] = [ @@ -36,12 +48,8 @@ extension AVCaptureDevice.Format { "colorSpaces": supportedColorSpaces.map(\.descriptor), "supportsVideoHDR": isVideoHDRSupported, "supportsPhotoHDR": false, - "frameRateRanges": videoSupportedFrameRateRanges.map { - [ - "minFrameRate": $0.minFrameRate, - "maxFrameRate": $0.maxFrameRate, - ] - }, + "minFps": minFrameRate, + "maxFps": maxFrameRate, "pixelFormat": CMFormatDescriptionGetMediaSubType(formatDescription).toString(), ] diff --git a/ios/Frame Processor/FrameHostObject.mm b/ios/Frame Processor/FrameHostObject.mm index aa427a5f89..89a92587b3 100644 --- a/ios/Frame Processor/FrameHostObject.mm +++ b/ios/Frame Processor/FrameHostObject.mm @@ -23,6 +23,7 @@ result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored"))); result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp"))); result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isDrawable"))); + result.push_back(jsi::PropNameID::forUtf8(rt, std::string("pixelFormat"))); // Conversion result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString"))); result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer"))); @@ -154,6 +155,19 @@ auto seconds = static_cast(CMTimeGetSeconds(timestamp)); return jsi::Value(seconds * 1000.0); } + if (name == "pixelFormat") { + auto format = CMSampleBufferGetFormatDescription(frame.buffer); + auto mediaType = CMFormatDescriptionGetMediaSubType(format); + switch (mediaType) { + case kCVPixelFormatType_32BGRA: + return jsi::String::createFromUtf8(runtime, "rgb"); + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + return jsi::String::createFromUtf8(runtime, "yuv"); + default: + return jsi::String::createFromUtf8(runtime, "unknown"); + } + } if (name == "bytesPerRow") { auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer); diff --git a/src/CameraDevice.ts b/src/CameraDevice.ts index e3a1ca0a53..b696160c18 100644 --- a/src/CameraDevice.ts +++ b/src/CameraDevice.ts @@ -99,11 +99,6 @@ export type AutoFocusSystem = 'contrast-detection' | 'phase-detection' | 'none'; */ export type VideoStabilizationMode = 'off' | 'standard' | 'cinematic' | 'cinematic-extended' | 'auto'; -export interface FrameRateRange { - minFrameRate: number; - maxFrameRate: number; -} - /** * A Camera Device's video format. Do not create instances of this type yourself, only use {@linkcode Camera.getAvailableCameraDevices | Camera.getAvailableCameraDevices()}. */ @@ -161,9 +156,13 @@ export interface CameraDeviceFormat { */ supportsPhotoHDR: boolean; /** - * All available frame rate ranges. You can query this to find the highest frame rate available + * The minum frame rate this Format needs to run at. High resolution formats often run at lower frame rates. + */ + minFps: number; + /** + * The maximum frame rate this Format is able to run at. High resolution formats often run at lower frame rates. */ - frameRateRanges: FrameRateRange[]; + maxFps: number; /** * Specifies this format's auto focus system. */ diff --git a/src/Frame.ts b/src/Frame.ts index cc1375366a..86de098ec8 100644 --- a/src/Frame.ts +++ b/src/Frame.ts @@ -1,5 +1,6 @@ import type { SkCanvas, SkPaint } from '@shopify/react-native-skia'; import type { Orientation } from './Orientation'; +import { PixelFormat } from './PixelFormat'; /** * A single frame, as seen by the camera. @@ -40,6 +41,10 @@ export interface Frame { * consideration when running a frame processor. See also: `isMirrored` */ orientation: Orientation; + /** + * Represents the pixel-format of the Frame. + */ + pixelFormat: PixelFormat; /** * Get the underlying data of the Frame as a uint8 array buffer. diff --git a/src/PixelFormat.ts b/src/PixelFormat.ts index 731f3eaf48..789daf0527 100644 --- a/src/PixelFormat.ts +++ b/src/PixelFormat.ts @@ -1,7 +1,7 @@ /** * Represents the pixel format of a `Frame`. - * * `420v`: 420 YpCbCr 8 Bi-Planar Video Range - * * `420f`: 420 YpCbCr 8 Bi-Planar Full Range - * * `x420`: 420 YpCbCr 10 Bi-Planar Video Range + * - `yuv`: Frame is in YUV pixel-format (_Bi-Planar Component Y'CbCr 8-bit 4:2:0, either full-range (luma=[0,255] chroma=[1,255]) or video-range (luma=[16,235]_) + * - `rgb`: Frame is in RGB pixel-format (BGRA 8-bit) + * - `unknown`: Frame has unknown pixel-format. */ -export type PixelFormat = '420f' | '420v' | 'x420'; +export type PixelFormat = 'yuv' | 'rgb' | 'unknown'; diff --git a/src/utils/FormatFilter.ts b/src/utils/FormatFilter.ts index 7a8ae6720c..70576b9d71 100644 --- a/src/utils/FormatFilter.ts +++ b/src/utils/FormatFilter.ts @@ -1,5 +1,5 @@ import { Dimensions } from 'react-native'; -import type { CameraDevice, CameraDeviceFormat, FrameRateRange } from '../CameraDevice'; +import type { CameraDevice, CameraDeviceFormat } from '../CameraDevice'; /** * Compares two devices by the following criteria: @@ -69,17 +69,3 @@ export const sortFormats = (left: CameraDeviceFormat, right: CameraDeviceFormat) return rightPoints - leftPoints; }; - -/** - * Returns `true` if the given Frame Rate Range (`range`) contains the given frame rate (`fps`) - * - * @param {FrameRateRange} range The range to check if the given `fps` are included in - * @param {number} fps The FPS to check if the given `range` supports. - * @example - * ```ts - * // get all formats that support 60 FPS - * const formatsWithHighFps = useMemo(() => device.formats.filter((f) => f.frameRateRanges.some((r) => frameRateIncluded(r, 60))), [device.formats]) - * ``` - * @method - */ -export const frameRateIncluded = (range: FrameRateRange, fps: number): boolean => fps >= range.minFrameRate && fps <= range.maxFrameRate; From 1ef1a50f75d2d46efc7b4f89aaf45f455d373186 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 15:24:28 +0200 Subject: [PATCH 011/180] Remove `isHighestPhotoQualitySupported` --- .../java/com/mrousavy/camera/utils/CameraDeviceDetails.kt | 3 --- ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift | 8 -------- ios/Extensions/AVCaptureDevice.Format+toDictionary.swift | 4 ---- src/CameraDevice.ts | 6 ------ 4 files changed, 21 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 5dd72b5547..a881c631f5 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -125,14 +125,11 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val } private fun buildFormatMap(outputSize: Size, outputFormat: Int, fpsRange: Range): ReadableMap { - val highResSizes = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) cameraConfig.getHighResolutionOutputSizes(outputFormat) else null) ?: emptyArray() - val map = Arguments.createMap() map.putInt("photoHeight", outputSize.height) map.putInt("photoWidth", outputSize.width) map.putInt("videoHeight", outputSize.height) map.putInt("videoWidth", outputSize.width) - map.putBoolean("isHighestPhotoQualitySupported", highResSizes.contains(outputSize)) map.putInt("maxISO", isoRange.upper) map.putInt("minISO", isoRange.lower) map.putDouble("fieldOfView", getFieldOfView()) diff --git a/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift b/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift index fa17f03dfc..2fd9b6f21b 100644 --- a/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift +++ b/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift @@ -85,14 +85,6 @@ extension AVCaptureDevice.Format { } } - if #available(iOS 13.0, *) { - if let isHighestPhotoQualitySupported = filter.value(forKey: "isHighestPhotoQualitySupported") as? Bool { - if self.isHighestPhotoQualitySupported != isHighestPhotoQualitySupported { - return false - } - } - } - return true } } diff --git a/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift b/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift index b728ccf47f..c5d3900dca 100644 --- a/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift +++ b/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift @@ -53,10 +53,6 @@ extension AVCaptureDevice.Format { "pixelFormat": CMFormatDescriptionGetMediaSubType(formatDescription).toString(), ] - if #available(iOS 13.0, *) { - dict["isHighestPhotoQualitySupported"] = self.isHighestPhotoQualitySupported - } - return dict } } diff --git a/src/CameraDevice.ts b/src/CameraDevice.ts index b696160c18..cd2304c4f7 100644 --- a/src/CameraDevice.ts +++ b/src/CameraDevice.ts @@ -119,12 +119,6 @@ export interface CameraDeviceFormat { * The video resolution's width */ videoWidth: number; - /** - * A boolean value specifying whether this format supports the highest possible photo quality that can be delivered on the current platform. - * - * @platform iOS 13.0+ - */ - isHighestPhotoQualitySupported?: boolean; /** * Maximum supported ISO value */ From 159c3b284acefcc8b80fc1dbaec1f5669fcc6759 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 15:29:47 +0200 Subject: [PATCH 012/180] Remove `colorSpace` The native platforms will use the best / most accurate colorSpace by default anyways. --- .../java/com/mrousavy/camera/CameraView.kt | 2 - .../com/mrousavy/camera/CameraViewManager.kt | 7 --- .../camera/utils/CameraDeviceDetails.kt | 13 ++---- docs/docs/guides/FORMATS.mdx | 1 - ios/CameraError.swift | 6 --- ios/CameraView+AVCaptureSession.swift | 10 +---- ios/CameraView.swift | 4 +- ios/CameraViewManager.m | 1 - ...AVCaptureDevice.Format+matchesFilter.swift | 7 --- .../AVCaptureDevice.Format+toDictionary.swift | 1 - .../AVCaptureColorSpace+descriptor.swift | 44 ------------------- ios/VisionCamera.xcodeproj/project.pbxproj | 4 -- src/CameraDevice.ts | 43 ------------------ src/CameraProps.ts | 8 +--- 14 files changed, 6 insertions(+), 145 deletions(-) delete mode 100644 ios/Parsers/AVCaptureColorSpace+descriptor.swift diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 2b0bc45794..daf1955075 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -40,7 +40,6 @@ import kotlin.math.min // TODO: configureSession() enableDepthData // TODO: configureSession() enableHighQualityPhotos // TODO: configureSession() enablePortraitEffectsMatteDelivery -// TODO: configureSession() colorSpace // CameraView+RecordVideo // TODO: Better startRecording()/stopRecording() (promise + callback, wait for TurboModules/JSI) @@ -83,7 +82,6 @@ class CameraView(context: Context) : FrameLayout(context) { var format: ReadableMap? = null var fps: Int? = null var hdr: Boolean? = null // nullable bool - var colorSpace: String? = null var lowLightBoost: Boolean? = null // nullable bool var previewType: String = "native" // other props diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt index 9ebf665b87..4b3e3b947b 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt @@ -120,13 +120,6 @@ class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManage view.lowLightBoost = lowLightBoost } - @ReactProp(name = "colorSpace") - fun setColorSpace(view: CameraView, colorSpace: String?) { - if (view.colorSpace != colorSpace) - addChangedPropToTransaction(view, "colorSpace") - view.colorSpace = colorSpace - } - @ReactProp(name = "isActive") fun setIsActive(view: CameraView, isActive: Boolean) { if (view.isActive != isActive) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index a881c631f5..f9bd189070 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -75,12 +75,6 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val return false } - private fun createColorSpaces(): ReadableArray { - val array = Arguments.createArray() - array.pushString("yuv") - return array - } - private fun createStabilizationModes(): ReadableArray { val array = Arguments.createArray() val videoStabilizationModes = digitalStabilizationModes.plus(opticalStabilizationModes) @@ -130,17 +124,16 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val map.putInt("photoWidth", outputSize.width) map.putInt("videoHeight", outputSize.height) map.putInt("videoWidth", outputSize.width) - map.putInt("maxISO", isoRange.upper) map.putInt("minISO", isoRange.lower) + map.putInt("maxISO", isoRange.upper) + map.putInt("minFps", fpsRange.lower) + map.putInt("maxFps", fpsRange.upper) map.putDouble("fieldOfView", getFieldOfView()) - map.putArray("colorSpaces", createColorSpaces()) map.putBoolean("supportsVideoHDR", supportsVideoHdr) map.putBoolean("supportsPhotoHDR", supportsPhotoHdr) map.putString("autoFocusSystem", "contrast-detection") // TODO: Is this wrong? map.putArray("videoStabilizationModes", createStabilizationModes()) map.putString("pixelFormat", parseImageFormat(outputFormat)) - map.putInt("minFps", fpsRange.lower) - map.putInt("maxFps", fpsRange.upper) return map } diff --git a/docs/docs/guides/FORMATS.mdx b/docs/docs/guides/FORMATS.mdx index 60d9bdce80..1cce09fceb 100644 --- a/docs/docs/guides/FORMATS.mdx +++ b/docs/docs/guides/FORMATS.mdx @@ -120,7 +120,6 @@ Other props that depend on the `format`: * `fps`: Specifies the frame rate to use * `hdr`: Enables HDR photo or video capture and preview * `lowLightBoost`: Enables a night-mode/low-light-boost for photo or video capture and preview -* `colorSpace`: Uses the specified color-space for photo or video capture and preview (iOS only since Android only uses `YUV`) * `videoStabilizationMode`: Specifies the video stabilization mode to use for this camera device diff --git a/ios/CameraError.swift b/ios/CameraError.swift index 46ff64b5fc..cc7d189144 100644 --- a/ios/CameraError.swift +++ b/ios/CameraError.swift @@ -112,7 +112,6 @@ enum FormatError { case invalidFps(fps: Int) case invalidHdr case invalidFormat - case invalidColorSpace(colorSpace: String) var code: String { switch self { @@ -122,8 +121,6 @@ enum FormatError { return "invalid-fps" case .invalidHdr: return "invalid-hdr" - case .invalidColorSpace: - return "invalid-color-space" } } @@ -135,9 +132,6 @@ enum FormatError { return "The given format cannot run at \(fps) FPS! Make sure your FPS is lower than `format.maxFps` but higher than `format.minFps`." case .invalidHdr: return "The currently selected format does not support HDR capture! Make sure you select a format which includes `supportsPhotoHDR`!" - case let .invalidColorSpace(colorSpace): - return "The currently selected format does not support the colorSpace \"\(colorSpace)\"! " + - "Make sure you select a format which `colorSpaces` includes \"\(colorSpace)\"!" } } } diff --git a/ios/CameraView+AVCaptureSession.swift b/ios/CameraView+AVCaptureSession.swift index 5d289ab02d..8dbc37cb3c 100644 --- a/ios/CameraView+AVCaptureSession.swift +++ b/ios/CameraView+AVCaptureSession.swift @@ -134,7 +134,7 @@ extension CameraView { // pragma MARK: Configure Device /** - Configures the Video Device with the given FPS, HDR and ColorSpace. + Configures the Video Device with the given FPS and HDR modes. */ final func configureDevice() { ReactLogger.log(level: .info, message: "Configuring Device...") @@ -182,14 +182,6 @@ extension CameraView { device.automaticallyEnablesLowLightBoostWhenAvailable = lowLightBoost!.boolValue } } - if let colorSpace = colorSpace as String? { - guard let avColorSpace = try? AVCaptureColorSpace(string: colorSpace), - device.activeFormat.supportedColorSpaces.contains(avColorSpace) else { - invokeOnError(.format(.invalidColorSpace(colorSpace: colorSpace))) - return - } - device.activeColorSpace = avColorSpace - } device.unlockForConfiguration() ReactLogger.log(level: .info, message: "Device successfully configured!") diff --git a/ios/CameraView.swift b/ios/CameraView.swift index 8b2868d9f6..c528c6ee00 100644 --- a/ios/CameraView.swift +++ b/ios/CameraView.swift @@ -29,8 +29,7 @@ private let propsThatRequireReconfiguration = ["cameraId", "previewType"] private let propsThatRequireDeviceReconfiguration = ["fps", "hdr", - "lowLightBoost", - "colorSpace"] + "lowLightBoost"] // MARK: - CameraView @@ -51,7 +50,6 @@ public final class CameraView: UIView { @objc var fps: NSNumber? @objc var hdr: NSNumber? // nullable bool @objc var lowLightBoost: NSNumber? // nullable bool - @objc var colorSpace: NSString? @objc var orientation: NSString? // other props @objc var isActive = false diff --git a/ios/CameraViewManager.m b/ios/CameraViewManager.m index ee79095ccd..d0481fe43c 100644 --- a/ios/CameraViewManager.m +++ b/ios/CameraViewManager.m @@ -37,7 +37,6 @@ @interface RCT_EXTERN_REMAP_MODULE(CameraView, CameraViewManager, RCTViewManager RCT_EXPORT_VIEW_PROPERTY(fps, NSNumber); RCT_EXPORT_VIEW_PROPERTY(hdr, NSNumber); // nullable bool RCT_EXPORT_VIEW_PROPERTY(lowLightBoost, NSNumber); // nullable bool -RCT_EXPORT_VIEW_PROPERTY(colorSpace, NSString); RCT_EXPORT_VIEW_PROPERTY(videoStabilizationMode, NSString); // other props RCT_EXPORT_VIEW_PROPERTY(torch, NSString); diff --git a/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift b/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift index 2fd9b6f21b..47f5ecb962 100644 --- a/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift +++ b/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift @@ -54,13 +54,6 @@ extension AVCaptureDevice.Format { return false } } - if let colorSpaces = filter.value(forKey: "colorSpaces") as? [String] { - let avColorSpaces = colorSpaces.map { try? AVCaptureColorSpace(string: $0) } - let allColorSpacesIncluded = supportedColorSpaces.allSatisfy { avColorSpaces.contains($0) } - if !allColorSpacesIncluded { - return false - } - } if let minFps = filter.value(forKey: "minFps") as? NSNumber { if minFrameRate != Float64(minFps.doubleValue) { return false diff --git a/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift b/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift index c5d3900dca..81008d0539 100644 --- a/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift +++ b/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift @@ -45,7 +45,6 @@ extension AVCaptureDevice.Format { "minISO": minISO, "fieldOfView": videoFieldOfView, "maxZoom": videoMaxZoomFactor, - "colorSpaces": supportedColorSpaces.map(\.descriptor), "supportsVideoHDR": isVideoHDRSupported, "supportsPhotoHDR": false, "minFps": minFrameRate, diff --git a/ios/Parsers/AVCaptureColorSpace+descriptor.swift b/ios/Parsers/AVCaptureColorSpace+descriptor.swift deleted file mode 100644 index 13a403b140..0000000000 --- a/ios/Parsers/AVCaptureColorSpace+descriptor.swift +++ /dev/null @@ -1,44 +0,0 @@ -// -// AVCaptureColorSpace+descriptor.swift -// mrousavy -// -// Created by Marc Rousavy on 19.12.20. -// Copyright © 2020 mrousavy. All rights reserved. -// - -import AVFoundation - -extension AVCaptureColorSpace { - init(string: String) throws { - switch string { - case "hlg-bt2020": - if #available(iOS 14.1, *) { - self = .HLG_BT2020 - } else { - throw EnumParserError.unsupportedOS(supportedOnOS: "14.1") - } - return - case "p3-d65": - self = .P3_D65 - return - case "srgb": - self = .sRGB - return - default: - throw EnumParserError.invalidValue - } - } - - var descriptor: String { - switch self { - case .HLG_BT2020: - return "hlg-bt2020" - case .P3_D65: - return "p3-d65" - case .sRGB: - return "srgb" - default: - fatalError("AVCaptureDevice.Position has unknown state.") - } - } -} diff --git a/ios/VisionCamera.xcodeproj/project.pbxproj b/ios/VisionCamera.xcodeproj/project.pbxproj index 7c5ecf5f74..a2bb4091f6 100644 --- a/ios/VisionCamera.xcodeproj/project.pbxproj +++ b/ios/VisionCamera.xcodeproj/project.pbxproj @@ -46,7 +46,6 @@ B887519F25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift */; }; B88751A025E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517B25E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift */; }; B88751A125E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517C25E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift */; }; - B88751A225E0102000DB86D6 /* AVCaptureColorSpace+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517D25E0102000DB86D6 /* AVCaptureColorSpace+descriptor.swift */; }; B88751A325E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517E25E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift */; }; B88751A425E0102000DB86D6 /* AVCaptureDevice.Format.AutoFocusSystem+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517F25E0102000DB86D6 /* AVCaptureDevice.Format.AutoFocusSystem+descriptor.swift */; }; B88751A525E0102000DB86D6 /* CameraView+Focus.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887518025E0102000DB86D6 /* CameraView+Focus.swift */; }; @@ -129,7 +128,6 @@ B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.DeviceType+descriptor.swift"; sourceTree = ""; }; B887517B25E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVAuthorizationStatus+descriptor.swift"; sourceTree = ""; }; B887517C25E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.Position+descriptor.swift"; sourceTree = ""; }; - B887517D25E0102000DB86D6 /* AVCaptureColorSpace+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureColorSpace+descriptor.swift"; sourceTree = ""; }; B887517E25E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.FlashMode+descriptor.swift"; sourceTree = ""; }; B887517F25E0102000DB86D6 /* AVCaptureDevice.Format.AutoFocusSystem+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.Format.AutoFocusSystem+descriptor.swift"; sourceTree = ""; }; B887518025E0102000DB86D6 /* CameraView+Focus.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = ""; }; @@ -255,7 +253,6 @@ B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift */, B887517B25E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift */, B887517C25E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift */, - B887517D25E0102000DB86D6 /* AVCaptureColorSpace+descriptor.swift */, B887517E25E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift */, B887517F25E0102000DB86D6 /* AVCaptureDevice.Format.AutoFocusSystem+descriptor.swift */, B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */, @@ -407,7 +404,6 @@ B887518625E0102000DB86D6 /* CameraView+RecordVideo.swift in Sources */, B81BE1BF26B936FF002696CC /* AVCaptureDevice.Format+videoDimensions.swift in Sources */, B8DB3BCA263DC4D8004C18D7 /* RecordingSession.swift in Sources */, - B88751A225E0102000DB86D6 /* AVCaptureColorSpace+descriptor.swift in Sources */, B83D5EE729377117000AFD2F /* NativePreviewView.swift in Sources */, B887518925E0102000DB86D6 /* Collection+safe.swift in Sources */, B887519125E0102000DB86D6 /* AVCaptureDevice.Format+toDictionary.swift in Sources */, diff --git a/src/CameraDevice.ts b/src/CameraDevice.ts index cd2304c4f7..1092af831f 100644 --- a/src/CameraDevice.ts +++ b/src/CameraDevice.ts @@ -42,43 +42,6 @@ export const parsePhysicalDeviceTypes = ( throw new Error(`Invalid physical device type combination! ${physicalDeviceTypes.join(' + ')}`); }; -/** - * Indicates a format's color space. - * - * #### The following colorspaces are available on iOS: - * * `"srgb"`: The sGRB color space. - * * `"p3-d65"`: The P3 D65 wide color space which uses Illuminant D65 as the white point - * * `"hlg-bt2020"`: The BT2020 wide color space which uses Illuminant D65 as the white point and Hybrid Log-Gamma as the transfer function - * - * > See ["AVCaptureColorSpace"](https://developer.apple.com/documentation/avfoundation/avcapturecolorspace) for more information. - * - * #### The following colorspaces are available on Android: - * * `"yuv"`: The Multi-plane Android YCbCr color space. (YUV 420_888, 422_888 or 444_888) - * * `"jpeg"`: The compressed JPEG color space. - * * `"jpeg-depth"`: The compressed JPEG color space including depth data. - * * `"raw"`: The Camera's RAW sensor color space. (Single-channel Bayer-mosaic image, usually 16 bit) - * * `"heic"`: The compressed HEIC color space. - * * `"private"`: The Android private opaque image format. (The choices of the actual format and pixel data layout are entirely up to the device-specific and framework internal implementations, and may vary depending on use cases even for the same device. These buffers are not directly accessible to the application) - * * `"depth-16"`: The Android dense depth image format (16 bit) - * * `"unknown"`: Placeholder for an unknown image/pixel format. [Edit this file](https://github.com/mrousavy/react-native-vision-camera/edit/main/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt) to add a name for the unknown format. - * - * > See ["Android Color Formats"](https://jbit.net/Android_Colors/) for more information. - */ -export type ColorSpace = - // ios - | 'hlg-bt2020' - | 'p3-d65' - | 'srgb' - // android - | 'yuv' - | 'jpeg' - | 'jpeg-depth' - | 'raw' - | 'heic' - | 'private' - | 'depth-16' - | 'unknown'; - /** * Indicates a format's autofocus system. * @@ -135,12 +98,6 @@ export interface CameraDeviceFormat { * The maximum zoom factor (e.g. `128`) */ maxZoom: number; - /** - * The available color spaces. - * - * Note: On Android, this will always be only `["yuv"]` - */ - colorSpaces: ColorSpace[]; /** * Specifies whether this format supports HDR mode for video capture */ diff --git a/src/CameraProps.ts b/src/CameraProps.ts index 3e1f7de218..56c0c58d4c 100644 --- a/src/CameraProps.ts +++ b/src/CameraProps.ts @@ -1,5 +1,5 @@ import type { ViewProps } from 'react-native'; -import type { CameraDevice, CameraDeviceFormat, ColorSpace, VideoStabilizationMode } from './CameraDevice'; +import type { CameraDevice, CameraDeviceFormat, VideoStabilizationMode } from './CameraDevice'; import type { CameraRuntimeError } from './CameraError'; import type { DrawableFrame, Frame } from './Frame'; import type { Orientation } from './Orientation'; @@ -115,12 +115,6 @@ export interface CameraProps extends ViewProps { * Requires `format` to be set. */ lowLightBoost?: boolean; - /** - * Specifies the color space to use for this camera device. Make sure the given `format` contains the given `colorSpace`. - * - * Requires `format` to be set. - */ - colorSpace?: ColorSpace; /** * Specifies the video stabilization mode to use for this camera device. Make sure the given `format` contains the given `videoStabilizationMode`. * From c52b81b81efac57f074d8c2d478446f6d9e86a67 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 15:35:34 +0200 Subject: [PATCH 013/180] HDR --- .../main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index f9bd189070..737e20159b 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -33,7 +33,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private val supportsLowLightBoost = extensions.contains(4 /* TODO: CameraExtensionCharacteristics.EXTENSION_NIGHT */) private val lensFacing = characteristics.get(CameraCharacteristics.LENS_FACING)!! private val hasFlash = characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE) ?: false - private val focalLengths = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) ?: FloatArray(0) + private val focalLengths = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) ?: floatArrayOf(35f /* 35mm default */) private val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! private val name = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) else null) ?: "${parseLensFacing(lensFacing)} (${cameraId})" @@ -70,6 +70,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val val availableProfiles = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES) ?: DynamicRangeProfiles(LongArray(0)) return availableProfiles.supportedProfiles.contains(DynamicRangeProfiles.HLG10) + || availableProfiles.supportedProfiles.contains(DynamicRangeProfiles.HDR10) } } return false From 1766ea3693d145fa49b98e27feb4d093c12027d2 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 16:23:40 +0200 Subject: [PATCH 014/180] Check from format --- .../java/com/mrousavy/camera/CameraView.kt | 59 ++++++++++++------- .../CameraDevice+createCaptureSession.kt | 43 +++++++++++++- .../camera/utils/Size+closestToOrMax.kt | 11 ++++ 3 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index daf1955075..32fc864c34 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -13,6 +13,7 @@ import android.hardware.camera2.params.StreamConfigurationMap import android.media.ImageReader import android.os.Build import android.util.Log +import android.util.Size import android.view.* import android.widget.FrameLayout import androidx.core.content.ContextCompat @@ -25,6 +26,7 @@ import com.mrousavy.camera.utils.SessionType import com.mrousavy.camera.utils.SurfaceOutput import com.mrousavy.camera.utils.createCaptureSession import com.mrousavy.camera.parsers.parseCameraError +import com.mrousavy.camera.parsers.parseHardwareLevel import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlin.math.max @@ -238,6 +240,12 @@ class CameraView(context: Context) : FrameLayout(context) { val characteristics = cameraManager.getCameraCharacteristics(camera.id) val isMirrored = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! + val supports3UseCases = hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY + && hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED + // If both photo and video/frame-processor is enabled but the Camera device only supports one of those use-cases, + // we need to fall back to Snapshot capture instead of actual photo capture since we cannot attach both photo and video. + val useSnapshotForPhotoCapture = !supports3UseCases && (photo == true && (video == true || enableFrameProcessor)) // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f @@ -245,29 +253,36 @@ class CameraView(context: Context) : FrameLayout(context) { val outputs = arrayListOf() if (photo == true) { - // Photo output: High quality still images - val format = ImageFormat.JPEG - // TODO: Let user configure photoSize with format (or new builder API) - val photoSize = config.getOutputSizes(format).maxBy { it.height * it.width } - val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, format, 1) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() - Log.d(TAG, "Photo captured! ${image.width} x ${image.height}") - image.close() - }, CameraQueues.cameraQueue.handler) - - Log.i(TAG, "Creating ${photoSize.width}x${photoSize.height} photo output. (Format: $format)") - val photoOutput = SurfaceOutput(imageReader.surface, isMirrored, OutputType.PHOTO) - outputs.add(photoOutput) + if (useSnapshotForPhotoCapture) { + Log.i(TAG, "The Camera Device ${camera.id} is running at hardware-level ${parseHardwareLevel(hardwareLevel)}. " + + "It does not support running both a photo and a video pipeline at the same time, so instead of adding a photo pipeline " + + "VisionCamera will just take snapshots of the video pipeline when calling takePhoto().") + } else { + // Photo output: High quality still images + val pixelFormat = ImageFormat.JPEG + val format = this.format + val targetSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null + val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(targetSize) + val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, pixelFormat, 1) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() + Log.d(TAG, "Photo captured! ${image.width} x ${image.height}") + image.close() + }, CameraQueues.cameraQueue.handler) + + Log.i(TAG, "Creating ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") + val photoOutput = SurfaceOutput(imageReader.surface, OutputType.PHOTO, isMirrored) + outputs.add(photoOutput) + } } if (video == true || enableFrameProcessor) { // Video or Frame Processor output: High resolution repeating images - val format = getVideoFormat(config) - config.inputFormats - // TODO: Let user configure videoSize with format (or new builder API) - val videoSize = config.getOutputSizes(format).maxBy { it.height * it.width } - val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, format, 2) + val pixelFormat = getVideoFormat(config) + val format = this.format + val targetSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null + val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(targetSize) + val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, pixelFormat, 2) imageReader.setOnImageAvailableListener({ reader -> val image = reader.acquireNextImage() if (image == null) { @@ -278,14 +293,14 @@ class CameraView(context: Context) : FrameLayout(context) { onFrame(frame) }, CameraQueues.videoQueue.handler) - Log.i(TAG, "Creating ${videoSize.width}x${videoSize.height} video output. (Format: $format)") - val videoOutput = SurfaceOutput(imageReader.surface, isMirrored, OutputType.VIDEO) + Log.i(TAG, "Creating ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") + val videoOutput = SurfaceOutput(imageReader.surface, OutputType.VIDEO, isMirrored) outputs.add(videoOutput) } if (previewType == "native") { // Preview output: Low resolution repeating images - val previewOutput = SurfaceOutput(previewView.holder.surface, isMirrored, OutputType.PREVIEW) + val previewOutput = SurfaceOutput(previewView.holder.surface, OutputType.PREVIEW, isMirrored) outputs.add(previewOutput) } diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 5df64354aa..78c2ed729b 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -1,13 +1,16 @@ package com.mrousavy.camera.utils +import android.content.res.Resources import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration +import android.media.CamcorderProfile import android.os.Build import android.util.Log +import android.util.Size import android.view.Surface import com.mrousavy.camera.CameraQueues import com.mrousavy.camera.CameraView @@ -47,8 +50,8 @@ enum class OutputType { } data class SurfaceOutput(val surface: Surface, + val outputType: OutputType, val isMirrored: Boolean = false, - val outputType: OutputType? = null, val dynamicRangeProfile: Long? = null) { val isRepeating: Boolean get() = outputType == OutputType.VIDEO || outputType == OutputType.PREVIEW || outputType == OutputType.VIDEO_AND_PREVIEW @@ -63,6 +66,9 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu } } } + // See https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture + // According to the Android Documentation, devices with LEVEL_3 or FULL support can do 4 use-cases. + // LIMITED or LEGACY devices can't do it. val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! if (hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3 || hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL) { return true @@ -71,6 +77,35 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu return false } +fun getMaxRecordResolution(cameraId: String): Size { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + val profiles = CamcorderProfile.getAll(cameraId, CamcorderProfile.QUALITY_HIGH) + if (profiles != null) { + val highestProfile = profiles.videoProfiles.maxBy { it.width * it.height } + return Size(highestProfile.width, highestProfile.height) + } + } + // fallback: old API + val cameraIdInt = cameraId.toIntOrNull() + val camcorderProfile = if (cameraIdInt != null) { + CamcorderProfile.get(cameraIdInt, CamcorderProfile.QUALITY_HIGH) + } else { + CamcorderProfile.get(CamcorderProfile.QUALITY_HIGH) + } + return Size(camcorderProfile.videoFrameWidth, camcorderProfile.videoFrameHeight) +} + +fun getMaxPreviewResolution(): Size { + val display = Resources.getSystem().displayMetrics + // According to Android documentation, "PREVIEW" size is always limited to 1920x1080 + return Size(1920.coerceAtMost(display.widthPixels), 1080.coerceAtMost(display.widthPixels)) +} + +fun getMaxMaximumResolution(format: Int, characteristics: CameraCharacteristics): Size { + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + return config.getOutputSizes(format).maxBy { it.width * it.height } +} + suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { return suspendCoroutine { continuation -> @@ -84,6 +119,9 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sess } } + val recordSize = getMaxRecordResolution(this.id) + val previewSize = getMaxPreviewResolution() + val characteristics = cameraManager.getCameraCharacteristics(this.id) val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! Log.i(CameraView.TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") @@ -91,10 +129,11 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sess if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { val outputConfigurations = outputs.map { val result = OutputConfiguration(it.surface) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { if (it.isMirrored) result.mirrorMode = OutputConfiguration.MIRROR_MODE_H if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile - if (it.outputType != null && supportsOutputType(characteristics, it.outputType)) { + if (supportsOutputType(characteristics, it.outputType)) { result.streamUseCase = it.outputType.toOutputType() } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt b/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt new file mode 100644 index 0000000000..5b7d9a3078 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt @@ -0,0 +1,11 @@ +package com.mrousavy.camera.utils + +import android.util.Size + +fun Array.closestToOrMax(size: Size?): Size { + return if (size != null) { + this.minBy { (it.width - size.width) + (it.height - size.height) } + } else { + this.maxBy { it.width * it.height } + } +} From 87ef655926a2814ef940f5cbce4e6e39641f7797 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 16:27:20 +0200 Subject: [PATCH 015/180] fix --- .../camera/utils/CameraDevice+createCaptureSession.kt | 4 ++-- .../java/com/mrousavy/camera/utils/CameraDeviceDetails.kt | 3 ++- .../java/com/mrousavy/camera/utils/Size+closestToOrMax.kt | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 78c2ed729b..35a86e0693 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -80,8 +80,8 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu fun getMaxRecordResolution(cameraId: String): Size { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { val profiles = CamcorderProfile.getAll(cameraId, CamcorderProfile.QUALITY_HIGH) - if (profiles != null) { - val highestProfile = profiles.videoProfiles.maxBy { it.width * it.height } + val highestProfile = profiles?.videoProfiles?.maxBy { it.width * it.height } + if (highestProfile != null) { return Size(highestProfile.width, highestProfile.height) } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 737e20159b..43c48b7eb1 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -147,7 +147,8 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val val outputSizes = cameraConfig.getOutputSizes(outputFormat).toMutableList() if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { // High resolution Photo sizes that are not able to run at 20FPS+ - outputSizes.addAll(cameraConfig.getHighResolutionOutputSizes(outputFormat)) + val highResSizes = cameraConfig.getHighResolutionOutputSizes(outputFormat) + if (highResSizes != null) outputSizes.addAll(highResSizes) } outputSizes.forEach { outputSize -> val frameDuration = cameraConfig.getOutputMinFrameDuration(outputFormat, outputSize) diff --git a/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt b/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt index 5b7d9a3078..3d93f238e5 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt @@ -1,10 +1,11 @@ package com.mrousavy.camera.utils import android.util.Size +import kotlin.math.abs fun Array.closestToOrMax(size: Size?): Size { return if (size != null) { - this.minBy { (it.width - size.width) + (it.height - size.height) } + this.minBy { abs(it.width - size.width) + abs(it.height - size.height) } } else { this.maxBy { it.width * it.height } } From b37c1e8990b5df91d5b9e25de1be60d7442fd843 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 16:43:52 +0200 Subject: [PATCH 016/180] Remove `supportsParallelVideoProcessing` --- .../main/java/com/mrousavy/camera/Errors.kt | 3 --- .../camera/utils/CameraDeviceDetails.kt | 4 ---- docs/docs/guides/CAPTURING.mdx | 2 +- docs/docs/guides/DEVICES.mdx | 22 ------------------- ios/CameraViewManager.swift | 1 - src/CameraDevice.ts | 10 --------- src/CameraProps.ts | 4 +--- 7 files changed, 2 insertions(+), 44 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 27a5246019..74783ebd77 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -37,9 +37,6 @@ class InvalidTypeScriptUnionError(unionName: String, unionValue: String) : Camer class NoCameraDeviceError : CameraError("device", "no-device", "No device was set! Use `getAvailableCameraDevices()` to select a suitable Camera device.") class InvalidCameraDeviceError(cause: Throwable) : CameraError("device", "invalid-device", "The given Camera device could not be found for use-case binding!", cause) -class ParallelVideoProcessingNotSupportedError(cause: Throwable) : CameraError("device", "parallel-video-processing-not-supported", "The given LEGACY Camera device does not support parallel " + - "video processing (`video={true}` + `frameProcessor={...}`). Disable either `video` or `frameProcessor`. To find out if a device supports parallel video processing, check the `supportsParallelVideoProcessing` property on the CameraDevice. " + - "See https://react-native-vision-camera.com/docs/guides/devices#the-supportsparallelvideoprocessing-prop for more information.", cause) class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given format cannot run at $fps FPS! Make sure your FPS is lower than `format.maxFps` but higher than `format.minFps`.") class HdrNotContainedInFormatError : CameraError( diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 43c48b7eb1..050efe1122 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -51,9 +51,6 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private val supportsPhotoHdr = extensions.contains(3 /* TODO: CameraExtensionCharacteristics.EXTENSION_HDR */) private val supportsVideoHdr = getHasVideoHdr() - // see https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture - private val supportsParallelVideoProcessing = hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY && hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED - // get extensions (HDR, Night Mode, ..) private fun getSupportedExtensions(): List { return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { @@ -174,7 +171,6 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val map.putBoolean("hasFlash", hasFlash) map.putBoolean("hasTorch", hasFlash) map.putBoolean("isMultiCam", isMultiCam) - map.putBoolean("supportsParallelVideoProcessing", supportsParallelVideoProcessing) map.putBoolean("supportsRawCapture", supportsRawCapture) map.putBoolean("supportsDepthCapture", supportsDepthCapture) map.putBoolean("supportsLowLightBoost", supportsLowLightBoost) diff --git a/docs/docs/guides/CAPTURING.mdx b/docs/docs/guides/CAPTURING.mdx index a77fe2ea02..e7bab85d64 100644 --- a/docs/docs/guides/CAPTURING.mdx +++ b/docs/docs/guides/CAPTURING.mdx @@ -73,7 +73,7 @@ While taking snapshots is faster than taking photos, the resulting image has way ::: :::note -The `takeSnapshot` function also works with `photo={false}`. For this reason VisionCamera will automatically fall-back to snapshot capture if you are trying to use more use-cases than the Camera natively supports. (see ["The `supportsParallelVideoProcessing` prop"](/docs/guides/devices#the-supportsparallelvideoprocessing-prop)) +The `takeSnapshot` function also works with `photo={false}`. For this reason VisionCamera will automatically fall-back to snapshot capture if you are trying to use more use-cases than the Camera natively supports. (see ["The `hardwareLevel` prop"](/docs/api/interfaces/CameraDevice#hardwarelevel)) ::: ## Recording Videos diff --git a/docs/docs/guides/DEVICES.mdx b/docs/docs/guides/DEVICES.mdx index 0d3ff445f3..de5f43ccef 100644 --- a/docs/docs/guides/DEVICES.mdx +++ b/docs/docs/guides/DEVICES.mdx @@ -46,7 +46,6 @@ The most important properties are: * `neutralZoom`: The zoom factor where the camera is "neutral". For any wide-angle cameras this property might be the same as `minZoom`, where as for ultra-wide-angle cameras ("fish-eye") this might be a value higher than `minZoom` (e.g. `2`). It is recommended that you always start at `neutralZoom` and let the user manually zoom out to `minZoom` on demand. * `maxZoom`: The maximum available zoom factor. When you pass `zoom={1}` to the Camera, the `maxZoom` factor will be applied. * `formats`: A list of all available formats (See [Camera Formats](formats)) -* `supportsParallelVideoProcessing`: Determines whether this camera devices supports using Video Recordings and Frame Processors at the same time. (See [`supportsParallelVideoProcessing`](#the-supportsparallelvideoprocessing-prop)) * `supportsFocus`: Determines whether this camera device supports focusing (See [Focusing](focusing)) :::note @@ -113,27 +112,6 @@ function App() { } ``` -### The `supportsParallelVideoProcessing` prop - -Camera devices provide the [`supportsParallelVideoProcessing` property](/docs/api/interfaces/CameraDevice#supportsparallelvideoprocessing) which determines whether the device supports using Video Recordings (`video={true}`) and Frame Processors (`frameProcessor={...}`) at the same time. - -If this property is `false`, you can either enable `video`, or add a `frameProcessor`, but not both. - -* On iOS this value is always `true`. -* On newer Android devices this value is always `true`. -* On older Android devices this value is `false` if the Camera's hardware level is `LEGACY` or `LIMITED`, `true` otherwise. (See [`INFO_SUPPORTED_HARDWARE_LEVEL`](https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics#INFO_SUPPORTED_HARDWARE_LEVEL) or [the tables at "Regular capture"](https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture)) - -#### Examples - -* An app that only supports **taking photos** (e.g. a vintage Polaroid Camera app) works on every Camera device because the `supportsParallelVideoProcessing` only affects _video processing_. -* An app that supports **taking photos** and **videos** (e.g. a Camera app) works on every Camera device because only a single _video processing_ feature is used (`video`). -* An app that only uses **Frame Processors** (e.g. the "Hotdog/Not Hotdog detector" app) (no taking photos or videos) works on every Camera device because it only uses a single _video processing_ feature (`frameProcessor`). -* An app that uses **Frame Processors** and supports **taking photos** and **videos** (e.g. Snapchat, Instagram) only works on Camera devices where `supportsParallelVideoProcessing` is `true`. (iPhones and newer Android Phones) - -:::note -Actually the limitation also affects the `photo` feature, but VisionCamera will automatically fall-back to **Snapshot capture** if you are trying to use multiple features (`photo` + `video` + `frameProcessor`) and they are not natively supported. (See ["Taking Snapshots"](/docs/guides/capturing#taking-snapshots)) -::: -
#### 🚀 Next section: [Camera Lifecycle](lifecycle) diff --git a/ios/CameraViewManager.swift b/ios/CameraViewManager.swift index 472b209495..34a5a06c51 100644 --- a/ios/CameraViewManager.swift +++ b/ios/CameraViewManager.swift @@ -117,7 +117,6 @@ final class CameraViewManager: RCTViewManager { "neutralZoom": $0.neutralZoomFactor, "maxZoom": $0.maxAvailableVideoZoomFactor, "isMultiCam": $0.isMultiCam, - "supportsParallelVideoProcessing": true, "supportsDepthCapture": false, // TODO: supportsDepthCapture "supportsRawCapture": false, // TODO: supportsRawCapture "supportsLowLightBoost": $0.isLowLightBoostSupported, diff --git a/src/CameraDevice.ts b/src/CameraDevice.ts index 1092af831f..1ecbbe6b45 100644 --- a/src/CameraDevice.ts +++ b/src/CameraDevice.ts @@ -201,16 +201,6 @@ export interface CameraDevice { * See [the Camera Formats documentation](https://react-native-vision-camera.com/docs/guides/formats) for more information about Camera Formats. */ formats: CameraDeviceFormat[]; - /** - * Whether this camera device supports using Video Recordings (`video={true}`) and Frame Processors (`frameProcessor={...}`) at the same time. See ["The `supportsParallelVideoProcessing` prop"](https://react-native-vision-camera.com/docs/guides/devices#the-supportsparallelvideoprocessing-prop) for more information. - * - * If this property is `false`, you can only enable `video` or add a `frameProcessor`, but not both. - * - * * On iOS this value is always `true`. - * * On newer Android devices this value is always `true`. - * * On older Android devices this value is `false` if the Camera's hardware level is `LEGACY` or `LIMITED`, `true` otherwise. (See [`INFO_SUPPORTED_HARDWARE_LEVEL`](https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics#INFO_SUPPORTED_HARDWARE_LEVEL) or [the tables at "Regular capture"](https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture)) - */ - supportsParallelVideoProcessing: boolean; /** * Whether this camera device supports low light boost. */ diff --git a/src/CameraProps.ts b/src/CameraProps.ts index 56c0c58d4c..3a541c4316 100644 --- a/src/CameraProps.ts +++ b/src/CameraProps.ts @@ -52,7 +52,7 @@ export interface CameraProps extends ViewProps { /** * Enables **video capture** with the `startRecording` function (see ["Recording Videos"](https://react-native-vision-camera.com/docs/guides/capturing/#recording-videos)) * - * Note: If you want to use `video` and `frameProcessor` simultaneously, make sure [`supportsParallelVideoProcessing`](https://react-native-vision-camera.com/docs/guides/devices#the-supportsparallelvideoprocessing-prop) is `true`. + * Note: If both the `photo` and `video` properties are enabled at the same time and the device is running at a `hardwareLevel` of `'legacy'` or `'limited'`, VisionCamera _might_ use a lower resolution for video capture due to hardware constraints. */ video?: boolean; /** @@ -177,8 +177,6 @@ export interface CameraProps extends ViewProps { * * If {@linkcode previewType | previewType} is set to `"skia"`, you can draw content to the `Frame` using the react-native-skia API. * - * Note: If you want to use `video` and `frameProcessor` simultaneously, make sure [`supportsParallelVideoProcessing`](https://react-native-vision-camera.com/docs/guides/devices#the-supportsparallelvideoprocessing-prop) is `true`. - * * > See [the Frame Processors documentation](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors) for more information * * @example From 015f220b8f328b643c6c90ab9c8126b3541d04fd Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 18:06:32 +0200 Subject: [PATCH 017/180] Correctly return video/photo sizes on Android now. Finally --- .../java/com/mrousavy/camera/CameraView.kt | 96 ++++++++++--------- .../CameraDevice+createCaptureSession.kt | 6 +- .../camera/utils/CameraDeviceDetails.kt | 49 ++++++---- 3 files changed, 85 insertions(+), 66 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 32fc864c34..90aa7aa6da 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -29,6 +29,7 @@ import com.mrousavy.camera.parsers.parseCameraError import com.mrousavy.camera.parsers.parseHardwareLevel import com.mrousavy.camera.utils.* import kotlinx.coroutines.* +import java.lang.IllegalArgumentException import kotlin.math.max import kotlin.math.min @@ -231,7 +232,7 @@ class CameraView(context: Context) : FrameLayout(context) { }, null) } - private suspend fun configureCamera(camera: CameraDevice) { + private suspend fun configureCamera(camera: CameraDevice, isSecondTryAfterConfigureError: Boolean = false) { if (cameraSession != null) { // Close any existing Session cameraSession?.close() @@ -240,49 +241,30 @@ class CameraView(context: Context) : FrameLayout(context) { val characteristics = cameraManager.getCameraCharacteristics(camera.id) val isMirrored = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! - val supports3UseCases = hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY - && hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED - // If both photo and video/frame-processor is enabled but the Camera device only supports one of those use-cases, - // we need to fall back to Snapshot capture instead of actual photo capture since we cannot attach both photo and video. - val useSnapshotForPhotoCapture = !supports3UseCases && (photo == true && (video == true || enableFrameProcessor)) // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f - val outputs = arrayListOf() + val format = this.format - if (photo == true) { - if (useSnapshotForPhotoCapture) { - Log.i(TAG, "The Camera Device ${camera.id} is running at hardware-level ${parseHardwareLevel(hardwareLevel)}. " + - "It does not support running both a photo and a video pipeline at the same time, so instead of adding a photo pipeline " + - "VisionCamera will just take snapshots of the video pipeline when calling takePhoto().") - } else { - // Photo output: High quality still images - val pixelFormat = ImageFormat.JPEG - val format = this.format - val targetSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null - val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(targetSize) - val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, pixelFormat, 1) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() - Log.d(TAG, "Photo captured! ${image.width} x ${image.height}") - image.close() - }, CameraQueues.cameraQueue.handler) - - Log.i(TAG, "Creating ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") - val photoOutput = SurfaceOutput(imageReader.surface, OutputType.PHOTO, isMirrored) - outputs.add(photoOutput) - } + val videoPixelFormat = getVideoFormat(config) + val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null + val videoSize = config.getOutputSizes(videoPixelFormat).closestToOrMax(targetVideoSize) + + // TODO: Let user configure .JPEG, .RAW_SENSOR, .HEIC + val photoPixelFormat = ImageFormat.JPEG + val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null + var photoSize = config.getOutputSizes(photoPixelFormat).closestToOrMax(targetPhotoSize) + if (isSecondTryAfterConfigureError) { + Log.i(TAG, "Trying to configure Camera now with RECORD resolution..") + photoSize = videoSize } + val outputs = arrayListOf() + if (video == true || enableFrameProcessor) { // Video or Frame Processor output: High resolution repeating images - val pixelFormat = getVideoFormat(config) - val format = this.format - val targetSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null - val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(targetSize) - val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, pixelFormat, 2) + val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, videoPixelFormat, 2) imageReader.setOnImageAvailableListener({ reader -> val image = reader.acquireNextImage() if (image == null) { @@ -293,9 +275,24 @@ class CameraView(context: Context) : FrameLayout(context) { onFrame(frame) }, CameraQueues.videoQueue.handler) - Log.i(TAG, "Creating ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") + Log.i(TAG, "Creating ${videoSize.width}x${videoSize.height} video output. (Format: $videoPixelFormat)") val videoOutput = SurfaceOutput(imageReader.surface, OutputType.VIDEO, isMirrored) outputs.add(videoOutput) + // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing + } + + if (photo == true) { + // Photo output: High quality still images + val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, photoPixelFormat, 1) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireLatestImage() + Log.d(TAG, "Photo captured! ${image.width} x ${image.height}") + image.close() + }, CameraQueues.cameraQueue.handler) + + Log.i(TAG, "Creating ${photoSize.width}x${photoSize.height} photo output. (Format: $photoPixelFormat)") + val photoOutput = SurfaceOutput(imageReader.surface, OutputType.PHOTO, isMirrored) + outputs.add(photoOutput) } if (previewType == "native") { @@ -304,17 +301,26 @@ class CameraView(context: Context) : FrameLayout(context) { outputs.add(previewOutput) } - cameraSession = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, CameraQueues.cameraQueue) + try { + cameraSession = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, CameraQueues.cameraQueue) - // Start all repeating requests (Video, Frame Processor, Preview) - val captureRequest = camera.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) - outputs.forEach { output -> - if (output.isRepeating) captureRequest.addTarget(output.surface) + // Start all repeating requests (Video, Frame Processor, Preview) + val captureRequest = camera.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) + outputs.forEach { output -> + if (output.isRepeating) captureRequest.addTarget(output.surface) + } + cameraSession!!.setRepeatingRequest(captureRequest.build(), null, null) + + Log.i(TAG, "Successfully configured Camera Session!") + invokeOnInitialized() + } catch (e: IllegalArgumentException) { + if (!isSecondTryAfterConfigureError) { + Log.e(TAG, "Failed to configure Camera: Caught Illegal Argument exception (\"${e.message}\")! " + + "Retrying once with lower resolution...", e) + return configureCamera(camera, true) + } + throw e } - cameraSession!!.setRepeatingRequest(captureRequest.build(), null, null) - - Log.i(TAG, "Successfully configured Camera Session!") - invokeOnInitialized() } private fun getVideoFormat(config: StreamConfigurationMap): Int { diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 35a86e0693..2c005de06b 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -140,15 +140,15 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sess return@map result } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - // API >28 + // API >=28 val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) this.createCaptureSession(config) } else { - // API >24 + // API >=24 this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) } } else { - // API <23 + // API <24 this.createCaptureSession(outputs.map { it.surface }, callback, queue.handler) } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 050efe1122..f4af9fb2d8 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -1,5 +1,7 @@ package com.mrousavy.camera.utils +import android.graphics.ImageFormat +import android.graphics.PixelFormat import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraExtensionCharacteristics @@ -51,6 +53,9 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private val supportsPhotoHdr = extensions.contains(3 /* TODO: CameraExtensionCharacteristics.EXTENSION_HDR */) private val supportsVideoHdr = getHasVideoHdr() + private val videoFormat = ImageFormat.YUV_420_888 + private val imageFormat = ImageFormat.JPEG + // get extensions (HDR, Night Mode, ..) private fun getSupportedExtensions(): List { return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { @@ -116,12 +121,12 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val return 2 * atan(sensorSize.bigger / (focalLengths[0] * 2)) * (180 / PI) } - private fun buildFormatMap(outputSize: Size, outputFormat: Int, fpsRange: Range): ReadableMap { + private fun buildFormatMap(photoSize: Size, videoSize: Size, outputFormat: Int, fpsRange: Range): ReadableMap { val map = Arguments.createMap() - map.putInt("photoHeight", outputSize.height) - map.putInt("photoWidth", outputSize.width) - map.putInt("videoHeight", outputSize.height) - map.putInt("videoWidth", outputSize.width) + map.putInt("photoHeight", photoSize.height) + map.putInt("photoWidth", photoSize.width) + map.putInt("videoHeight", videoSize.height) + map.putInt("videoWidth", videoSize.width) map.putInt("minISO", isoRange.lower) map.putInt("maxISO", isoRange.upper) map.putInt("minFps", fpsRange.lower) @@ -135,23 +140,31 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val return map } + private fun getVideoSizes(): Array { + return cameraConfig.getOutputSizes(ImageFormat.YUV_420_888) ?: emptyArray() + } + private fun getPhotoSizes(): Array { + val sizes = cameraConfig.getOutputSizes(ImageFormat.JPEG) ?: emptyArray() + val highResSizes = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + cameraConfig.getHighResolutionOutputSizes(ImageFormat.JPEG) + } else { + null + } ?: emptyArray() + return sizes.plus(highResSizes) + } + private fun getFormats(): ReadableArray { val array = Arguments.createArray() - val outputFormats = cameraConfig.outputFormats - outputFormats.forEach { outputFormat -> - // Normal Video/Photo Sizes - val outputSizes = cameraConfig.getOutputSizes(outputFormat).toMutableList() - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - // High resolution Photo sizes that are not able to run at 20FPS+ - val highResSizes = cameraConfig.getHighResolutionOutputSizes(outputFormat) - if (highResSizes != null) outputSizes.addAll(highResSizes) - } - outputSizes.forEach { outputSize -> - val frameDuration = cameraConfig.getOutputMinFrameDuration(outputFormat, outputSize) - val maxFps = (1.0 / (frameDuration.toDouble() / 1000000000)).toInt() + val videoSizes = getVideoSizes() + val photoSizes = getPhotoSizes() + + videoSizes.forEach { videoSize -> + val frameDuration = cameraConfig.getOutputMinFrameDuration(videoFormat, videoSize) + val maxFps = (1.0 / (frameDuration.toDouble() / 1000000000)).toInt() - val map = buildFormatMap(outputSize, outputFormat, Range(1, maxFps)) + photoSizes.forEach { photoSize -> + val map = buildFormatMap(photoSize, videoSize, videoFormat, Range(1, maxFps)) array.pushMap(map) } } From 3eb7a48f25acb86c904ea16aeb9d7e63e7fa0750 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 18:14:01 +0200 Subject: [PATCH 018/180] Log all Device props --- .../camera/utils/CameraDeviceDetails.kt | 33 +++++++++++++++++++ example/src/CameraPage.tsx | 2 ++ 2 files changed, 35 insertions(+) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index f4af9fb2d8..f1d01d3721 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -193,6 +193,39 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val map.putDouble("neutralZoom", 1.0) // Zoom is always relative to 1.0 on Android map.putString("hardwareLevel", parseHardwareLevel(hardwareLevel)) + val array = Arguments.createArray() + cameraConfig.outputFormats.forEach { f -> + val str = when (f) { + ImageFormat.YUV_420_888 -> "YUV_420_888" + ImageFormat.YUV_422_888 -> "YUV_422_888" + ImageFormat.YUV_444_888 -> "YUV_444_888" + ImageFormat.JPEG -> "JPEG" + ImageFormat.DEPTH16 -> "DEPTH16" + ImageFormat.DEPTH_JPEG -> "DEPTH_JPEG" + ImageFormat.FLEX_RGBA_8888 -> "FLEX_RGBA_8888" + ImageFormat.FLEX_RGB_888 -> "FLEX_RGB_888" + ImageFormat.YUY2 -> "YUY2" + ImageFormat.Y8 -> "Y8" + ImageFormat.YV12 -> "YV12" + ImageFormat.HEIC -> "HEIC" + ImageFormat.PRIVATE -> "PRIVATE" + ImageFormat.RAW_PRIVATE -> "RAW_PRIVATE" + ImageFormat.RAW_SENSOR -> "RAW_SENSOR" + ImageFormat.RAW10 -> "RAW10" + ImageFormat.RAW12 -> "RAW12" + PixelFormat.RGB_888 -> "RGB_888" + PixelFormat.RGBA_8888 -> "RGBA_8888" + PixelFormat.RGBX_8888 -> "RGBX_8888" + ImageFormat.NV16 -> "NV16" + ImageFormat.NV21 -> "NV21" + ImageFormat.UNKNOWN -> "UNKNOWN" + ImageFormat.YCBCR_P010 -> "YCBCR_P010" + else -> "unknown ($f)" + } + array.pushString(str) + } + map.putArray("pixelFormats", array) + map.putArray("formats", getFormats()) return map diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index 73c5d6edd9..9f76f62404 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -224,6 +224,8 @@ export function CameraPage({ navigation }: Props): React.ReactElement { console.log(frame.timestamp, frame.toString(), frame.pixelFormat); }, []); + console.log(JSON.stringify(device)); + return ( {device != null && ( From 1ff083dc78be7919fdeaf1a0a23faf0ce3759f21 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 18:54:45 +0200 Subject: [PATCH 019/180] Log if optimized usecase is used --- .../CameraDevice+createCaptureSession.kt | 33 +------------------ example/src/CameraPage.tsx | 4 +-- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 2c005de06b..6dd53798f5 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -77,35 +77,6 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu return false } -fun getMaxRecordResolution(cameraId: String): Size { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - val profiles = CamcorderProfile.getAll(cameraId, CamcorderProfile.QUALITY_HIGH) - val highestProfile = profiles?.videoProfiles?.maxBy { it.width * it.height } - if (highestProfile != null) { - return Size(highestProfile.width, highestProfile.height) - } - } - // fallback: old API - val cameraIdInt = cameraId.toIntOrNull() - val camcorderProfile = if (cameraIdInt != null) { - CamcorderProfile.get(cameraIdInt, CamcorderProfile.QUALITY_HIGH) - } else { - CamcorderProfile.get(CamcorderProfile.QUALITY_HIGH) - } - return Size(camcorderProfile.videoFrameWidth, camcorderProfile.videoFrameHeight) -} - -fun getMaxPreviewResolution(): Size { - val display = Resources.getSystem().displayMetrics - // According to Android documentation, "PREVIEW" size is always limited to 1920x1080 - return Size(1920.coerceAtMost(display.widthPixels), 1080.coerceAtMost(display.widthPixels)) -} - -fun getMaxMaximumResolution(format: Int, characteristics: CameraCharacteristics): Size { - val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - return config.getOutputSizes(format).maxBy { it.width * it.height } -} - suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { return suspendCoroutine { continuation -> @@ -119,9 +90,6 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sess } } - val recordSize = getMaxRecordResolution(this.id) - val previewSize = getMaxPreviewResolution() - val characteristics = cameraManager.getCameraCharacteristics(this.id) val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! Log.i(CameraView.TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") @@ -135,6 +103,7 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sess if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile if (supportsOutputType(characteristics, it.outputType)) { result.streamUseCase = it.outputType.toOutputType() + Log.i(CameraView.TAG, "Using optimized stream use case \"${it.outputType.name}\" (${result.streamUseCase})..") } } return@map result diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index 9f76f62404..847628b7ec 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -192,7 +192,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { if (device != null && format != null) { console.log( `Re-rendering camera page with ${isActive ? 'active' : 'inactive'} camera. ` + - `Device: "${device.name}" (${format.photoWidth}x${format.photoHeight} @ ${fps}fps)`, + `Device: "${device.name}" (${format.photoWidth}x${format.photoHeight} photo / ${format.videoWidth}x${format.videoHeight} video @ ${fps}fps)`, ); } else { console.log('re-rendering camera page without active camera'); @@ -224,8 +224,6 @@ export function CameraPage({ navigation }: Props): React.ReactElement { console.log(frame.timestamp, frame.toString(), frame.pixelFormat); }, []); - console.log(JSON.stringify(device)); - return ( {device != null && ( From 3fc817793ecd7ac531d0246a0fd6f54b08b7d518 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 19:07:31 +0200 Subject: [PATCH 020/180] Cleanup --- .../java/com/mrousavy/camera/CameraView.kt | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 90aa7aa6da..1a8166bc17 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -233,12 +233,11 @@ class CameraView(context: Context) : FrameLayout(context) { } private suspend fun configureCamera(camera: CameraDevice, isSecondTryAfterConfigureError: Boolean = false) { - if (cameraSession != null) { - // Close any existing Session - cameraSession?.close() - } + // Close any existing Session + cameraSession?.close() val characteristics = cameraManager.getCameraCharacteristics(camera.id) + // TODO: Mirroring is probably done automatically, can we remove this flag? val isMirrored = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! @@ -275,7 +274,7 @@ class CameraView(context: Context) : FrameLayout(context) { onFrame(frame) }, CameraQueues.videoQueue.handler) - Log.i(TAG, "Creating ${videoSize.width}x${videoSize.height} video output. (Format: $videoPixelFormat)") + Log.i(TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $videoPixelFormat)") val videoOutput = SurfaceOutput(imageReader.surface, OutputType.VIDEO, isMirrored) outputs.add(videoOutput) // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing @@ -290,7 +289,7 @@ class CameraView(context: Context) : FrameLayout(context) { image.close() }, CameraQueues.cameraQueue.handler) - Log.i(TAG, "Creating ${photoSize.width}x${photoSize.height} photo output. (Format: $photoPixelFormat)") + Log.i(TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $photoPixelFormat)") val photoOutput = SurfaceOutput(imageReader.surface, OutputType.PHOTO, isMirrored) outputs.add(photoOutput) } @@ -298,6 +297,7 @@ class CameraView(context: Context) : FrameLayout(context) { if (previewType == "native") { // Preview output: Low resolution repeating images val previewOutput = SurfaceOutput(previewView.holder.surface, OutputType.PREVIEW, isMirrored) + Log.i(TAG, "Adding native preview view output.") outputs.add(previewOutput) } @@ -315,8 +315,12 @@ class CameraView(context: Context) : FrameLayout(context) { invokeOnInitialized() } catch (e: IllegalArgumentException) { if (!isSecondTryAfterConfigureError) { + // See https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture + // According to the Android Documentation, it is not guaranteed that a device can stream Images in maximum resolution + // for both photo capture (JPEG) and video (YUV) capture at the same time. + // If this is the case, a compromise has to be made. We try to configure the session with a lower photo resolution. Log.e(TAG, "Failed to configure Camera: Caught Illegal Argument exception (\"${e.message}\")! " + - "Retrying once with lower resolution...", e) + "Retrying once with lower photo resolution...", e) return configureCamera(camera, true) } throw e @@ -329,11 +333,8 @@ class CameraView(context: Context) : FrameLayout(context) { if (formats.contains(ImageFormat.YUV_420_888)) { return ImageFormat.YUV_420_888 } - if (formats.contains(PixelFormat.RGB_888)) { - return PixelFormat.RGB_888; - } - Log.w(TAG, "Couldn't find YUV_420_888 or RGB_888 format for Video " + - "Recording, using unknown format instead.. (${formats[0]})") + Log.w(TAG, "Couldn't find YUV_420_888 format for Video Streams, " + + "using unknown format instead.. (${formats[0]})") return formats[0] } } From 0fd6c03f54c7566cb5592053720c4a8743aba92e Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 20:06:00 +0200 Subject: [PATCH 021/180] Configure Camera Input only once --- ios/CameraView+AVCaptureSession.swift | 39 +++++++++++++++------------ ios/CameraView.swift | 15 ++++++----- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/ios/CameraView+AVCaptureSession.swift b/ios/CameraView+AVCaptureSession.swift index 8dbc37cb3c..f95539e75a 100644 --- a/ios/CameraView+AVCaptureSession.swift +++ b/ios/CameraView+AVCaptureSession.swift @@ -14,32 +14,19 @@ import Foundation */ extension CameraView { // pragma MARK: Configure Capture Session - - /** - Configures the Capture Session. - */ - final func configureCaptureSession() { - ReactLogger.log(level: .info, message: "Configuring Session...") - isReady = false - - #if targetEnvironment(simulator) - invokeOnError(.device(.notAvailableOnSimulator)) - return - #endif - - guard cameraId != nil else { + + final func configureCameraInput() { + guard let cameraId = cameraId as? String else { invokeOnError(.device(.noDevice)) return } - let cameraId = self.cameraId! as String ReactLogger.log(level: .info, message: "Initializing Camera with device \(cameraId)...") captureSession.beginConfiguration() defer { captureSession.commitConfiguration() } - - // pragma MARK: Capture Session Inputs + // Video Input do { if let videoDeviceInput = videoDeviceInput { @@ -61,6 +48,24 @@ extension CameraView { invokeOnError(.device(.invalid)) return } + } + + /** + Configures the Capture Session. + */ + final func configureCaptureSession() { + ReactLogger.log(level: .info, message: "Configuring Session...") + isReady = false + + #if targetEnvironment(simulator) + invokeOnError(.device(.notAvailableOnSimulator)) + return + #endif + + captureSession.beginConfiguration() + defer { + captureSession.commitConfiguration() + } // pragma MARK: Capture Session Outputs diff --git a/ios/CameraView.swift b/ios/CameraView.swift index c528c6ee00..fea34e39cd 100644 --- a/ios/CameraView.swift +++ b/ios/CameraView.swift @@ -19,8 +19,7 @@ import UIKit // CameraView+TakePhoto // TODO: Photo HDR -private let propsThatRequireReconfiguration = ["cameraId", - "enableDepthData", +private let propsThatRequireReconfiguration = ["enableDepthData", "enableHighQualityPhotos", "enablePortraitEffectsMatteDelivery", "photo", @@ -173,12 +172,13 @@ public final class CameraView: UIView { // pragma MARK: Props updating override public final func didSetProps(_ changedProps: [String]!) { ReactLogger.log(level: .info, message: "Updating \(changedProps.count) prop(s)...") - let shouldReconfigure = changedProps.contains { propsThatRequireReconfiguration.contains($0) } - let shouldReconfigureFormat = shouldReconfigure || changedProps.contains("format") + let shouldReconfigure = changedProps.contains("cameraId") + let shouldReconfigureSession = shouldReconfigure || changedProps.contains { propsThatRequireReconfiguration.contains($0) } + let shouldReconfigureFormat = shouldReconfigureSession || changedProps.contains("format") let shouldReconfigureDevice = shouldReconfigureFormat || changedProps.contains { propsThatRequireDeviceReconfiguration.contains($0) } let shouldReconfigureAudioSession = changedProps.contains("audio") - let willReconfigure = shouldReconfigure || shouldReconfigureFormat || shouldReconfigureDevice + let willReconfigure = shouldReconfigureSession || shouldReconfigureFormat || shouldReconfigureDevice let shouldCheckActive = willReconfigure || changedProps.contains("isActive") || captureSession.isRunning != isActive let shouldUpdateTorch = willReconfigure || changedProps.contains("torch") || shouldCheckActive @@ -197,7 +197,7 @@ public final class CameraView: UIView { } } - if shouldReconfigure || + if shouldReconfigureSession || shouldReconfigureAudioSession || shouldCheckActive || shouldUpdateTorch || @@ -209,6 +209,9 @@ public final class CameraView: UIView { CameraQueues.cameraQueue.async { // Video Configuration if shouldReconfigure { + self.configureCameraInput() + } + if shouldReconfigureSession { self.configureCaptureSession() } if shouldReconfigureFormat { From 8b93cfa5a3da0c32aee7c831d9d03e54aed33adc Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 20:06:26 +0200 Subject: [PATCH 022/180] Revert "Configure Camera Input only once" This reverts commit 0fd6c03f54c7566cb5592053720c4a8743aba92e. --- ios/CameraView+AVCaptureSession.swift | 39 ++++++++++++--------------- ios/CameraView.swift | 15 +++++------ 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/ios/CameraView+AVCaptureSession.swift b/ios/CameraView+AVCaptureSession.swift index f95539e75a..8dbc37cb3c 100644 --- a/ios/CameraView+AVCaptureSession.swift +++ b/ios/CameraView+AVCaptureSession.swift @@ -14,19 +14,32 @@ import Foundation */ extension CameraView { // pragma MARK: Configure Capture Session - - final func configureCameraInput() { - guard let cameraId = cameraId as? String else { + + /** + Configures the Capture Session. + */ + final func configureCaptureSession() { + ReactLogger.log(level: .info, message: "Configuring Session...") + isReady = false + + #if targetEnvironment(simulator) + invokeOnError(.device(.notAvailableOnSimulator)) + return + #endif + + guard cameraId != nil else { invokeOnError(.device(.noDevice)) return } + let cameraId = self.cameraId! as String ReactLogger.log(level: .info, message: "Initializing Camera with device \(cameraId)...") captureSession.beginConfiguration() defer { captureSession.commitConfiguration() } - + + // pragma MARK: Capture Session Inputs // Video Input do { if let videoDeviceInput = videoDeviceInput { @@ -48,24 +61,6 @@ extension CameraView { invokeOnError(.device(.invalid)) return } - } - - /** - Configures the Capture Session. - */ - final func configureCaptureSession() { - ReactLogger.log(level: .info, message: "Configuring Session...") - isReady = false - - #if targetEnvironment(simulator) - invokeOnError(.device(.notAvailableOnSimulator)) - return - #endif - - captureSession.beginConfiguration() - defer { - captureSession.commitConfiguration() - } // pragma MARK: Capture Session Outputs diff --git a/ios/CameraView.swift b/ios/CameraView.swift index fea34e39cd..c528c6ee00 100644 --- a/ios/CameraView.swift +++ b/ios/CameraView.swift @@ -19,7 +19,8 @@ import UIKit // CameraView+TakePhoto // TODO: Photo HDR -private let propsThatRequireReconfiguration = ["enableDepthData", +private let propsThatRequireReconfiguration = ["cameraId", + "enableDepthData", "enableHighQualityPhotos", "enablePortraitEffectsMatteDelivery", "photo", @@ -172,13 +173,12 @@ public final class CameraView: UIView { // pragma MARK: Props updating override public final func didSetProps(_ changedProps: [String]!) { ReactLogger.log(level: .info, message: "Updating \(changedProps.count) prop(s)...") - let shouldReconfigure = changedProps.contains("cameraId") - let shouldReconfigureSession = shouldReconfigure || changedProps.contains { propsThatRequireReconfiguration.contains($0) } - let shouldReconfigureFormat = shouldReconfigureSession || changedProps.contains("format") + let shouldReconfigure = changedProps.contains { propsThatRequireReconfiguration.contains($0) } + let shouldReconfigureFormat = shouldReconfigure || changedProps.contains("format") let shouldReconfigureDevice = shouldReconfigureFormat || changedProps.contains { propsThatRequireDeviceReconfiguration.contains($0) } let shouldReconfigureAudioSession = changedProps.contains("audio") - let willReconfigure = shouldReconfigureSession || shouldReconfigureFormat || shouldReconfigureDevice + let willReconfigure = shouldReconfigure || shouldReconfigureFormat || shouldReconfigureDevice let shouldCheckActive = willReconfigure || changedProps.contains("isActive") || captureSession.isRunning != isActive let shouldUpdateTorch = willReconfigure || changedProps.contains("torch") || shouldCheckActive @@ -197,7 +197,7 @@ public final class CameraView: UIView { } } - if shouldReconfigureSession || + if shouldReconfigure || shouldReconfigureAudioSession || shouldCheckActive || shouldUpdateTorch || @@ -209,9 +209,6 @@ public final class CameraView: UIView { CameraQueues.cameraQueue.async { // Video Configuration if shouldReconfigure { - self.configureCameraInput() - } - if shouldReconfigureSession { self.configureCaptureSession() } if shouldReconfigureFormat { From fe8904550b9bb3a658a099685b2699ebab9612a1 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 2 Aug 2023 21:14:17 +0200 Subject: [PATCH 023/180] Extract Camera configuration --- .../java/com/mrousavy/camera/CameraQueues.kt | 10 +- .../java/com/mrousavy/camera/CameraSession.kt | 145 ++++++++++++ .../java/com/mrousavy/camera/CameraView.kt | 215 +++++++----------- .../com/mrousavy/camera/CameraViewManager.kt | 14 ++ .../frameprocessor/VisionCameraScheduler.java | 3 +- .../parsers/VideoStabilizationMode+String.kt | 9 + .../CameraDevice+createCaptureSession.kt | 3 + .../camera/utils/CameraManager+openCamera.kt | 52 +++++ src/CameraProps.ts | 5 +- 9 files changed, 315 insertions(+), 141 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/CameraSession.kt create mode 100644 android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt index ea90e8cb3f..9360d4a735 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt @@ -3,7 +3,10 @@ package com.mrousavy.camera import android.os.Handler import android.os.HandlerThread import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.android.asCoroutineDispatcher import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.asExecutor +import java.util.concurrent.Executor import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import kotlin.coroutines.CoroutineContext @@ -15,17 +18,18 @@ class CameraQueues { } class CameraQueue(name: String) { - val executor: ExecutorService val handler: Handler val coroutineScope: CoroutineScope private val thread: HandlerThread + val executor: Executor init { thread = HandlerThread(name) thread.start() handler = Handler(thread.looper) - executor = Executors.newSingleThreadExecutor() - coroutineScope = CoroutineScope(executor.asCoroutineDispatcher()) + val coroutineDispatcher = handler.asCoroutineDispatcher() + coroutineScope = CoroutineScope(coroutineDispatcher) + executor = coroutineDispatcher.asExecutor() } protected fun finalize() { diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt new file mode 100644 index 0000000000..2aea9038e6 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -0,0 +1,145 @@ +package com.mrousavy.camera + +import android.graphics.ImageFormat +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.hardware.camera2.CaptureRequest +import android.media.Image +import android.media.ImageReader +import android.os.Build +import android.util.Log +import android.util.Range +import android.util.Size +import android.view.Surface +import com.mrousavy.camera.parsers.getVideoStabilizationMode +import com.mrousavy.camera.utils.OutputType +import com.mrousavy.camera.utils.SessionType +import com.mrousavy.camera.utils.SurfaceOutput +import com.mrousavy.camera.utils.closestToOrMax +import com.mrousavy.camera.utils.createCaptureSession +import java.io.Closeable + +data class PipelineConfiguration(val enabled: Boolean, + val callback: (image: Image) -> Unit, + val targetSize: Size? = null) + +class CameraSession(private val device: CameraDevice, + private val captureSession: CameraCaptureSession, + private val outputs: List): Closeable { + private var captureRequest: CaptureRequest = createCaptureRequestBuilder().build() + + private fun createCaptureRequestBuilder(): CaptureRequest.Builder { + val captureRequest = device.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) + outputs.forEach { output -> + if (output.isRepeating) captureRequest.addTarget(output.surface) + } + return captureRequest + } + + fun configureFormat(fps: Int? = null, + videoStabilizationMode: String? = null, + hdr: Boolean? = null, + lowLightBoost: Boolean? = null) { + val captureRequest = createCaptureRequestBuilder() + if (fps != null) { + captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) + } + if (videoStabilizationMode != null) { + captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, getVideoStabilizationMode(videoStabilizationMode)) + } + if (lowLightBoost == true) { + captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) + } + if (hdr == true) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP_MR1) { + captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR) + } + } + this.captureRequest = captureRequest.build() + } + + + fun startRunning() { + // Start all repeating requests (Video, Frame Processor, Preview) + captureSession.setRepeatingRequest(captureRequest, null, null) + } + + fun stopRunning() { + captureSession.stopRepeating() + } + + override fun close() { + stopRunning() + captureSession.close() + } + + companion object { + suspend fun createCameraSession(device: CameraDevice, + cameraManager: CameraManager, + photoPipeline: PipelineConfiguration? = null, + videoPipeline: PipelineConfiguration? = null, + previewSurface: Surface? = null): CameraSession { + val characteristics = cameraManager.getCameraCharacteristics(device.id) + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + + val outputs = arrayListOf() + + if (videoPipeline != null) { + // Video or Frame Processor output: High resolution repeating images + val pixelFormat = ImageFormat.YUV_420_888 + val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoPipeline.targetSize) + + val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, pixelFormat, 2) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() + if (image == null) { + Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") + return@setOnImageAvailableListener + } + + videoPipeline.callback(image) + }, CameraQueues.videoQueue.handler) + + Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") + val videoOutput = SurfaceOutput(imageReader.surface, OutputType.VIDEO) + outputs.add(videoOutput) + // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing + } + + if (photoPipeline != null) { + // Photo output: High quality still images + val pixelFormat = ImageFormat.JPEG + val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoPipeline.targetSize) + + val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, pixelFormat, 1) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireLatestImage() + image.use { + Log.d(CameraView.TAG, "Photo captured! ${image.width} x ${image.height}") + photoPipeline.callback(image) + } + }, CameraQueues.cameraQueue.handler) + + Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") + val photoOutput = SurfaceOutput(imageReader.surface, OutputType.PHOTO) + outputs.add(photoOutput) + } + + if (previewSurface != null) { + // Preview output: Low resolution repeating images + val previewOutput = SurfaceOutput(previewSurface, OutputType.PREVIEW) + Log.i(CameraView.TAG, "Adding native preview view output.") + outputs.add(previewOutput) + } + + val captureSession = device.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, CameraQueues.cameraQueue) + + Log.i(CameraView.TAG, "Successfully configured Camera Session!") + return CameraSession(device, captureSession, outputs) + } + + } +} + diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 1a8166bc17..130b900b38 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -13,23 +13,31 @@ import android.hardware.camera2.params.StreamConfigurationMap import android.media.ImageReader import android.os.Build import android.util.Log +import android.util.Range import android.util.Size import android.view.* import android.widget.FrameLayout import androidx.core.content.ContextCompat +import androidx.core.view.isVisible import androidx.lifecycle.* import com.facebook.react.bridge.* import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.parsers.getVideoStabilizationMode import com.mrousavy.camera.utils.OutputType import com.mrousavy.camera.utils.SessionType import com.mrousavy.camera.utils.SurfaceOutput import com.mrousavy.camera.utils.createCaptureSession import com.mrousavy.camera.parsers.parseCameraError import com.mrousavy.camera.parsers.parseHardwareLevel +import com.mrousavy.camera.parsers.parseVideoStabilizationMode import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import java.lang.IllegalArgumentException +import kotlin.coroutines.coroutineContext +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine import kotlin.math.max import kotlin.math.min @@ -60,13 +68,14 @@ import kotlin.math.min // TODO: takePhoto() enableAutoDistortionCorrection // TODO: takePhoto() return with jsi::Value Image reference for faster capture -@Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that. @SuppressLint("ClickableViewAccessibility", "ViewConstructor", "MissingPermission") class CameraView(context: Context) : FrameLayout(context) { companion object { const val TAG = "CameraView" - private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "fps", "hdr", "lowLightBoost", "photo", "video", "enableFrameProcessor") + private val propsThatRequireDeviceReconfiguration = arrayListOf("cameraId") + private val propsThatRequireSessionReconfiguration = arrayListOf("format", "photo", "video", "enableFrameProcessor") + private val propsThatRequireFormatReconfiguration = arrayListOf("fps", "hdr", "videoStabilizationMode", "lowLightBoost") private val arrayListOfZoom = arrayListOf("zoom") } @@ -84,6 +93,7 @@ class CameraView(context: Context) : FrameLayout(context) { // props that require format reconfiguring var format: ReadableMap? = null var fps: Int? = null + var videoStabilizationMode: String? = null var hdr: Boolean? = null // nullable bool var lowLightBoost: Boolean? = null // nullable bool var previewType: String = "native" @@ -98,7 +108,8 @@ class CameraView(context: Context) : FrameLayout(context) { private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager // session - private var cameraSession: CameraCaptureSession? = null + private var cameraDevice: CameraDevice? = null + private var cameraSession: CameraSession? = null private val previewView = SurfaceView(context) private var isPreviewSurfaceReady = false @@ -135,7 +146,6 @@ class CameraView(context: Context) : FrameLayout(context) { override fun surfaceCreated(holder: SurfaceHolder) { Log.i(TAG, "PreviewView Surface created!") isPreviewSurfaceReady = true - if (cameraId != null) configureSession() } override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { @@ -157,16 +167,16 @@ class CameraView(context: Context) : FrameLayout(context) { override fun onAttachedToWindow() { super.onAttachedToWindow() - // TODO: updateLifecycleState() if (!isMounted) { isMounted = true invokeOnViewReady() } + updateLifecycle() } override fun onDetachedFromWindow() { super.onDetachedFromWindow() - // TODO: updateLifecycleState() + updateLifecycle() } /** @@ -174,17 +184,34 @@ class CameraView(context: Context) : FrameLayout(context) { */ fun update(changedProps: ArrayList) { try { - val shouldReconfigureSession = changedProps.containsAny(propsThatRequireSessionReconfiguration) - val shouldReconfigureZoom = shouldReconfigureSession || changedProps.contains("zoom") - val shouldReconfigureTorch = shouldReconfigureSession || changedProps.contains("torch") - val shouldUpdateOrientation = shouldReconfigureSession || changedProps.contains("orientation") - - if (changedProps.contains("isActive")) { - // TODO: updateLifecycleState() - } - if (shouldReconfigureSession) { - // configureSession() + val shouldReconfigureDevice = changedProps.containsAny(propsThatRequireDeviceReconfiguration) + val shouldReconfigureSession = shouldReconfigureDevice || changedProps.containsAny(propsThatRequireSessionReconfiguration) + val shouldReconfigureFormat = shouldReconfigureSession || changedProps.containsAny(propsThatRequireFormatReconfiguration) + val shouldReconfigureZoom = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("zoom") + val shouldReconfigureTorch = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("torch") + val shouldUpdateOrientation = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("orientation") + val shouldCheckActive = shouldReconfigureFormat || changedProps.contains("isActive") + + CameraQueues.cameraQueue.coroutineScope.launch { + try { + if (shouldReconfigureDevice) { + configureDevice() + } + if (shouldReconfigureSession) { + configureSession() + } + if (shouldReconfigureFormat) { + configureFormat() + } + if (shouldCheckActive) { + updateLifecycle() + } + } catch (e: Throwable) { + Log.e(TAG, "Failed to configure Camera!", e) + invokeOnError(e) + } } + if (shouldReconfigureZoom) { val zoomClamped = max(min(zoom, maxZoom), minZoom) // TODO: camera!!.cameraControl.setZoomRatio(zoomClamped) @@ -202,139 +229,59 @@ class CameraView(context: Context) : FrameLayout(context) { } /** - * Configures the camera capture session. This should only be called when the camera device changes. + * Prepares the hardware Camera Device. (cameraId) */ - private fun configureSession() { - Log.i(TAG, "Configuring session...") + private suspend fun configureDevice() { + Log.i(TAG, "Configuring Camera Device...") if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { throw CameraPermissionError() } val cameraId = cameraId ?: throw NoCameraDeviceError() - Log.i(TAG, "Opening Camera $cameraId...") - cameraManager.openCamera(cameraId, object: CameraDevice.StateCallback() { - override fun onOpened(camera: CameraDevice) { - Log.i(TAG, "Successfully opened Camera Device $cameraId!") - CameraQueues.cameraQueue.coroutineScope.launch { - configureCamera(camera) - } - } - - override fun onDisconnected(camera: CameraDevice) { - Log.i(TAG, "Camera Device $cameraId has been disconnected! Waiting for reconnect to continue session..") - invokeOnError(CameraDisconnectedError(cameraId)) - } - - override fun onError(camera: CameraDevice, error: Int) { - Log.e(TAG, "Failed to open Camera Device $cameraId! Error: $error (${parseCameraError(error)})") - invokeOnError(CameraCannotBeOpenedError(cameraId, parseCameraError(error))) - } - }, null) + cameraDevice = cameraManager.openCamera(cameraId) } - private suspend fun configureCamera(camera: CameraDevice, isSecondTryAfterConfigureError: Boolean = false) { - // Close any existing Session - cameraSession?.close() - - val characteristics = cameraManager.getCameraCharacteristics(camera.id) - // TODO: Mirroring is probably done automatically, can we remove this flag? - val isMirrored = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT - val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - - // TODO: minZoom = camera!!.cameraInfo.zoomState.value?.minZoomRatio ?: 1f - // TODO: maxZoom = camera!!.cameraInfo.zoomState.value?.maxZoomRatio ?: 1f - - val format = this.format + private suspend fun configureSession() { + val cameraDevice = cameraDevice + if (cameraDevice == null) { + Log.w(TAG, "Tried to call configureSession() without a CameraDevice! Returning...") + return + } - val videoPixelFormat = getVideoFormat(config) + val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null - val videoSize = config.getOutputSizes(videoPixelFormat).closestToOrMax(targetVideoSize) - - // TODO: Let user configure .JPEG, .RAW_SENSOR, .HEIC - val photoPixelFormat = ImageFormat.JPEG val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null - var photoSize = config.getOutputSizes(photoPixelFormat).closestToOrMax(targetPhotoSize) - if (isSecondTryAfterConfigureError) { - Log.i(TAG, "Trying to configure Camera now with RECORD resolution..") - photoSize = videoSize - } - - val outputs = arrayListOf() - - if (video == true || enableFrameProcessor) { - // Video or Frame Processor output: High resolution repeating images - val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, videoPixelFormat, 2) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() - if (image == null) { - Log.w(TAG, "Failed to get new Image from ImageReader, dropping a Frame...") - return@setOnImageAvailableListener - } - val frame = Frame(image, System.currentTimeMillis(), inputRotation, isMirrored) + val previewSurface = if (previewType == "native") previewView.holder.surface else null + + cameraSession = CameraSession.createCameraSession( + cameraDevice, + cameraManager, + // Photo Pipeline + PipelineConfiguration(video == true, { + Log.i(TAG, "Captured an Image!") + }, targetPhotoSize), + // Video Pipeline + PipelineConfiguration(photo == true, { image -> + val frame = Frame(image, System.currentTimeMillis(), inputRotation, false) onFrame(frame) - }, CameraQueues.videoQueue.handler) - - Log.i(TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $videoPixelFormat)") - val videoOutput = SurfaceOutput(imageReader.surface, OutputType.VIDEO, isMirrored) - outputs.add(videoOutput) - // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing - } - - if (photo == true) { - // Photo output: High quality still images - val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, photoPixelFormat, 1) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireLatestImage() - Log.d(TAG, "Photo captured! ${image.width} x ${image.height}") - image.close() - }, CameraQueues.cameraQueue.handler) - - Log.i(TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $photoPixelFormat)") - val photoOutput = SurfaceOutput(imageReader.surface, OutputType.PHOTO, isMirrored) - outputs.add(photoOutput) - } - - if (previewType == "native") { - // Preview output: Low resolution repeating images - val previewOutput = SurfaceOutput(previewView.holder.surface, OutputType.PREVIEW, isMirrored) - Log.i(TAG, "Adding native preview view output.") - outputs.add(previewOutput) - } - - try { - cameraSession = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, CameraQueues.cameraQueue) - - // Start all repeating requests (Video, Frame Processor, Preview) - val captureRequest = camera.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) - outputs.forEach { output -> - if (output.isRepeating) captureRequest.addTarget(output.surface) - } - cameraSession!!.setRepeatingRequest(captureRequest.build(), null, null) + }, targetVideoSize), + // Preview Pipeline + previewSurface + ) + } - Log.i(TAG, "Successfully configured Camera Session!") - invokeOnInitialized() - } catch (e: IllegalArgumentException) { - if (!isSecondTryAfterConfigureError) { - // See https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture - // According to the Android Documentation, it is not guaranteed that a device can stream Images in maximum resolution - // for both photo capture (JPEG) and video (YUV) capture at the same time. - // If this is the case, a compromise has to be made. We try to configure the session with a lower photo resolution. - Log.e(TAG, "Failed to configure Camera: Caught Illegal Argument exception (\"${e.message}\")! " + - "Retrying once with lower photo resolution...", e) - return configureCamera(camera, true) - } - throw e - } + private fun configureFormat() { + cameraSession?.configureFormat(fps, videoStabilizationMode, hdr, lowLightBoost) } - private fun getVideoFormat(config: StreamConfigurationMap): Int { - val formats = config.outputFormats - Log.i(TAG, "Device supports ${formats.size} output formats: ${formats.joinToString(", ")}") - if (formats.contains(ImageFormat.YUV_420_888)) { - return ImageFormat.YUV_420_888 + private fun updateLifecycle() { + val cameraSession = cameraSession + if (isActive && isAttachedToWindow && cameraSession != null) { + Log.i(TAG, "Starting Camera Session...") + cameraSession.startRunning() + } else { + Log.i(TAG, "Stopping Camera Session...") + cameraSession?.stopRunning() } - Log.w(TAG, "Couldn't find YUV_420_888 format for Video Streams, " + - "using unknown format instead.. (${formats[0]})") - return formats[0] } } diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt index 4b3e3b947b..54caa80ea7 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt @@ -75,6 +75,20 @@ class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManage view.enableDepthData = enableDepthData } + @ReactProp(name = "videoStabilizationMode") + fun setVideoStabilizationMode(view: CameraView, videoStabilizationMode: String?) { + if (view.videoStabilizationMode != videoStabilizationMode) + addChangedPropToTransaction(view, "videoStabilizationMode") + view.videoStabilizationMode = videoStabilizationMode + } + + @ReactProp(name = "previewType") + fun setPreviewType(view: CameraView, previewType: String?) { + if (view.previewType != previewType) + addChangedPropToTransaction(view, "previewType") + view.previewType = previewType ?: "native" + } + @ReactProp(name = "enableHighQualityPhotos") fun setEnableHighQualityPhotos(view: CameraView, enableHighQualityPhotos: Boolean?) { if (view.enableHighQualityPhotos != enableHighQualityPhotos) diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java index 456430675c..f7b82b2c27 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java @@ -23,6 +23,7 @@ public VisionCameraScheduler() { @DoNotStrip private void scheduleTrigger() { CameraQueues.CameraQueue videoQueue = CameraQueues.Companion.getVideoQueue(); - videoQueue.getExecutor().submit(this::trigger); + // TODO: Make sure post(this::trigger) works. + videoQueue.getHandler().post(this::trigger); } } diff --git a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt index ff01b6bd41..21563084a9 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt @@ -10,3 +10,12 @@ fun parseVideoStabilizationMode(stabiliazionMode: Int): String { else -> "off" } } + +fun getVideoStabilizationMode(stabiliazionMode: String): Int { + return when (stabiliazionMode) { + "off" -> CONTROL_VIDEO_STABILIZATION_MODE_OFF + "standard" -> CONTROL_VIDEO_STABILIZATION_MODE_ON + "cinematic" -> 2 /* TODO: CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION */ + else -> CONTROL_VIDEO_STABILIZATION_MODE_OFF + } +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 6dd53798f5..ea3a6f8077 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -15,6 +15,9 @@ import android.view.Surface import com.mrousavy.camera.CameraQueues import com.mrousavy.camera.CameraView import com.mrousavy.camera.parsers.parseHardwareLevel +import kotlinx.coroutines.android.asCoroutineDispatcher +import kotlinx.coroutines.asExecutor +import java.util.concurrent.Executor import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt new file mode 100644 index 0000000000..26c9d0ebd8 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt @@ -0,0 +1,52 @@ +package com.mrousavy.camera.utils + +import android.annotation.SuppressLint +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.util.Log +import com.mrousavy.camera.CameraCannotBeOpenedError +import com.mrousavy.camera.CameraDisconnectedError +import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.CameraView +import com.mrousavy.camera.parsers.parseCameraError +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine + +@SuppressLint("MissingPermission") +suspend fun CameraManager.openCamera(cameraId: String): CameraDevice { + return suspendCoroutine { continuation -> + var didRun = false + val runOnce = { block: () -> Unit -> + if (!didRun) { + block() + didRun = true + } + } + Log.i(CameraView.TAG, "Opening Camera $cameraId...") + + + this.openCamera(cameraId, object: CameraDevice.StateCallback() { + override fun onOpened(device: CameraDevice) { + Log.i(CameraView.TAG, "Successfully opened Camera Device $cameraId!") + runOnce { + continuation.resume(device) + } + } + + override fun onDisconnected(camera: CameraDevice) { + Log.w(CameraView.TAG, "Camera Device $cameraId has been disconnected! Closing Camera..") + runOnce { + continuation.resumeWithException(CameraDisconnectedError(cameraId)) + } + } + + override fun onError(camera: CameraDevice, errorCode: Int) { + Log.e(CameraView.TAG, "Failed to open Camera Device $cameraId! Closing Camera.. Error: $errorCode (${parseCameraError(errorCode)})") + runOnce { + continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, parseCameraError(errorCode))) + } + } + }, CameraQueues.cameraQueue.handler) + } +} diff --git a/src/CameraProps.ts b/src/CameraProps.ts index 3a541c4316..35583ff226 100644 --- a/src/CameraProps.ts +++ b/src/CameraProps.ts @@ -116,10 +116,9 @@ export interface CameraProps extends ViewProps { */ lowLightBoost?: boolean; /** - * Specifies the video stabilization mode to use for this camera device. Make sure the given `format` contains the given `videoStabilizationMode`. + * Specifies the video stabilization mode to use. * - * Requires `format` to be set. - * @platform iOS + * Requires a `format` to be set that contains the given `videoStabilizationMode`. */ videoStabilizationMode?: VideoStabilizationMode; //#endregion From ee25982c26374242bbc5b514ab1cd5b163b2cd80 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 11:08:14 +0200 Subject: [PATCH 024/180] Try to reconfigure all --- .../java/com/mrousavy/camera/CameraView.kt | 20 +++++++++++++++++- .../camera/utils/CameraManager+openCamera.kt | 21 ++++++++++--------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 130b900b38..7540dd1e48 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -146,15 +146,19 @@ class CameraView(context: Context) : FrameLayout(context) { override fun surfaceCreated(holder: SurfaceHolder) { Log.i(TAG, "PreviewView Surface created!") isPreviewSurfaceReady = true + reconfigureAll() } override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { Log.i(TAG, "PreviewView Surface resized!") + isPreviewSurfaceReady = true + reconfigureAll() } override fun surfaceDestroyed(holder: SurfaceHolder) { Log.i(TAG, "PreviewView Surface destroyed!") isPreviewSurfaceReady = false + reconfigureAll() } }) addView(previewView) @@ -238,7 +242,12 @@ class CameraView(context: Context) : FrameLayout(context) { } val cameraId = cameraId ?: throw NoCameraDeviceError() - cameraDevice = cameraManager.openCamera(cameraId) + cameraDevice = cameraManager.openCamera(cameraId) { + Log.i(TAG, "Camera Closed!") + cameraSession?.close() + cameraSession = null + cameraDevice = null + } } private suspend fun configureSession() { @@ -284,4 +293,13 @@ class CameraView(context: Context) : FrameLayout(context) { cameraSession?.stopRunning() } } + + private fun reconfigureAll() { + CameraQueues.cameraQueue.coroutineScope.launch { + configureDevice() + configureSession() + configureFormat() + updateLifecycle() + } + } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt index 26c9d0ebd8..eaaf7e8e5b 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt @@ -14,37 +14,38 @@ import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine @SuppressLint("MissingPermission") -suspend fun CameraManager.openCamera(cameraId: String): CameraDevice { +suspend fun CameraManager.openCamera(cameraId: String, onClosed: () -> Unit): CameraDevice { return suspendCoroutine { continuation -> var didRun = false - val runOnce = { block: () -> Unit -> - if (!didRun) { - block() - didRun = true - } - } Log.i(CameraView.TAG, "Opening Camera $cameraId...") this.openCamera(cameraId, object: CameraDevice.StateCallback() { override fun onOpened(device: CameraDevice) { Log.i(CameraView.TAG, "Successfully opened Camera Device $cameraId!") - runOnce { + if (!didRun) { continuation.resume(device) + didRun = true } } override fun onDisconnected(camera: CameraDevice) { Log.w(CameraView.TAG, "Camera Device $cameraId has been disconnected! Closing Camera..") - runOnce { + if (!didRun) { continuation.resumeWithException(CameraDisconnectedError(cameraId)) + didRun = true + } else { + onClosed() } } override fun onError(camera: CameraDevice, errorCode: Int) { Log.e(CameraView.TAG, "Failed to open Camera Device $cameraId! Closing Camera.. Error: $errorCode (${parseCameraError(errorCode)})") - runOnce { + if (!didRun) { continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, parseCameraError(errorCode))) + didRun = true + } else { + onClosed() } } }, CameraQueues.cameraQueue.handler) From 882972dc1458e98918d684a944696474b43d83b3 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 13:17:03 +0200 Subject: [PATCH 025/180] Hook based --- .../java/com/mrousavy/camera/CameraSession.kt | 18 ++- .../java/com/mrousavy/camera/CameraView.kt | 147 ++++++++---------- .../com/mrousavy/camera/NativePreviewView.kt | 73 +++++++++ .../com/mrousavy/camera/hooks/HookListener.kt | 15 ++ .../mrousavy/camera/hooks/UseCameraDevice.kt | 74 +++++++++ .../camera/hooks/UseSurfaceViewSurface.kt | 38 +++++ .../mrousavy/camera/utils/Size+Extensions.kt | 37 +++++ .../camera/utils/Size+closestToOrMax.kt | 12 -- .../com/mrousavy/camera/utils/Size+rotated.kt | 17 -- 9 files changed, 315 insertions(+), 116 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/NativePreviewView.kt create mode 100644 android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt create mode 100644 android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt create mode 100644 android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt create mode 100644 android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/Size+rotated.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 2aea9038e6..fd81b8113d 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -20,6 +20,7 @@ import com.mrousavy.camera.utils.SurfaceOutput import com.mrousavy.camera.utils.closestToOrMax import com.mrousavy.camera.utils.createCaptureSession import java.io.Closeable +import java.lang.IllegalStateException data class PipelineConfiguration(val enabled: Boolean, val callback: (image: Image) -> Unit, @@ -62,12 +63,22 @@ class CameraSession(private val device: CameraDevice, fun startRunning() { - // Start all repeating requests (Video, Frame Processor, Preview) - captureSession.setRepeatingRequest(captureRequest, null, null) + Log.i(TAG, "Starting Camera Session...") + try { + // Start all repeating requests (Video, Frame Processor, Preview) + captureSession.setRepeatingRequest(captureRequest, null, null) + } catch (e: IllegalStateException) { + Log.w(TAG, "Failed to start Camera Session, this session is already closed.") + } } fun stopRunning() { - captureSession.stopRepeating() + Log.i(TAG, "Stopping Camera Session...") + try { + captureSession.stopRepeating() + } catch (e: IllegalStateException) { + Log.w(TAG, "Failed to stop Camera Session, this session is already closed.") + } } override fun close() { @@ -76,6 +87,7 @@ class CameraSession(private val device: CameraDevice, } companion object { + private const val TAG = "CameraSession" suspend fun createCameraSession(device: CameraDevice, cameraManager: CameraManager, photoPipeline: PipelineConfiguration? = null, diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 7540dd1e48..4a84c61ddd 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -5,39 +5,23 @@ import android.annotation.SuppressLint import android.content.Context import android.content.pm.PackageManager import android.content.res.Configuration -import android.graphics.ImageFormat -import android.graphics.PixelFormat -import android.hardware.camera2.* -import android.hardware.camera2.CameraDevice -import android.hardware.camera2.params.StreamConfigurationMap -import android.media.ImageReader -import android.os.Build +import android.hardware.camera2.CameraManager import android.util.Log -import android.util.Range import android.util.Size -import android.view.* +import android.view.Surface +import android.view.SurfaceView +import android.view.View import android.widget.FrameLayout import androidx.core.content.ContextCompat -import androidx.core.view.isVisible -import androidx.lifecycle.* -import com.facebook.react.bridge.* +import com.facebook.react.bridge.ReadableMap import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor -import com.mrousavy.camera.parsers.getVideoStabilizationMode -import com.mrousavy.camera.utils.OutputType -import com.mrousavy.camera.utils.SessionType -import com.mrousavy.camera.utils.SurfaceOutput -import com.mrousavy.camera.utils.createCaptureSession -import com.mrousavy.camera.parsers.parseCameraError -import com.mrousavy.camera.parsers.parseHardwareLevel -import com.mrousavy.camera.parsers.parseVideoStabilizationMode -import com.mrousavy.camera.utils.* -import kotlinx.coroutines.* -import java.lang.IllegalArgumentException -import kotlin.coroutines.coroutineContext -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlin.coroutines.suspendCoroutine +import com.mrousavy.camera.hooks.UseCameraDevice +import com.mrousavy.camera.hooks.UseSurfaceViewSurface +import com.mrousavy.camera.utils.containsAny +import com.mrousavy.camera.utils.displayRotation +import com.mrousavy.camera.utils.installHierarchyFitter +import kotlinx.coroutines.launch import kotlin.math.max import kotlin.math.min @@ -73,15 +57,19 @@ class CameraView(context: Context) : FrameLayout(context) { companion object { const val TAG = "CameraView" - private val propsThatRequireDeviceReconfiguration = arrayListOf("cameraId") - private val propsThatRequireSessionReconfiguration = arrayListOf("format", "photo", "video", "enableFrameProcessor") + private val propsThatRequirePreviewReconfiguration = arrayListOf("cameraId", "previewType") + private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "photo", "video", "enableFrameProcessor") private val propsThatRequireFormatReconfiguration = arrayListOf("fps", "hdr", "videoStabilizationMode", "lowLightBoost") private val arrayListOfZoom = arrayListOf("zoom") } // react properties // props that require reconfiguring - var cameraId: String? = null // this is actually not a react prop directly, but the result of setting device={} + var cameraId: String? = null + set(value) { + field = value + if (value != null) configureDevice() + } var enableDepthData = false var enableHighQualityPhotos: Boolean? = null var enablePortraitEffectsMatteDelivery = false @@ -108,10 +96,10 @@ class CameraView(context: Context) : FrameLayout(context) { private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager // session - private var cameraDevice: CameraDevice? = null + private var cameraDevice: UseCameraDevice? = null private var cameraSession: CameraSession? = null - private val previewView = SurfaceView(context) - private var isPreviewSurfaceReady = false + private var previewView: View? = null + private var previewSurface: UseSurfaceViewSurface? = null public var frameProcessor: FrameProcessor? = null @@ -141,27 +129,7 @@ class CameraView(context: Context) : FrameLayout(context) { init { this.installHierarchyFitter() - previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) - previewView.holder.addCallback(object : SurfaceHolder.Callback { - override fun surfaceCreated(holder: SurfaceHolder) { - Log.i(TAG, "PreviewView Surface created!") - isPreviewSurfaceReady = true - reconfigureAll() - } - - override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { - Log.i(TAG, "PreviewView Surface resized!") - isPreviewSurfaceReady = true - reconfigureAll() - } - - override fun surfaceDestroyed(holder: SurfaceHolder) { - Log.i(TAG, "PreviewView Surface destroyed!") - isPreviewSurfaceReady = false - reconfigureAll() - } - }) - addView(previewView) + setupPreviewView() } override fun onConfigurationChanged(newConfig: Configuration?) { @@ -183,24 +151,41 @@ class CameraView(context: Context) : FrameLayout(context) { updateLifecycle() } - /** - * Invalidate all React Props and reconfigure the device - */ + private fun setupPreviewView() { + val cameraId = cameraId ?: return + + if (previewType == "native") { + if (this.previewView is SurfaceView) return + removeView(this.previewView) + + val previewView = NativePreviewView(cameraManager, cameraId, context) + previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) + previewSurface = UseSurfaceViewSurface(previewView) { + Log.i(TAG, "PreviewView Surface changed!") + reconfigureAll() + } + addView(previewView) + this.previewView = previewView + } else { + throw Error("Skia is not yet implemented on Android!") + } + } + fun update(changedProps: ArrayList) { try { - val shouldReconfigureDevice = changedProps.containsAny(propsThatRequireDeviceReconfiguration) - val shouldReconfigureSession = shouldReconfigureDevice || changedProps.containsAny(propsThatRequireSessionReconfiguration) + val shouldReconfigurePreview = changedProps.containsAny(propsThatRequirePreviewReconfiguration) + val shouldReconfigureSession = shouldReconfigurePreview || changedProps.containsAny(propsThatRequireSessionReconfiguration) val shouldReconfigureFormat = shouldReconfigureSession || changedProps.containsAny(propsThatRequireFormatReconfiguration) val shouldReconfigureZoom = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("zoom") val shouldReconfigureTorch = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("torch") val shouldUpdateOrientation = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("orientation") val shouldCheckActive = shouldReconfigureFormat || changedProps.contains("isActive") + if (shouldReconfigurePreview) { + setupPreviewView() + } CameraQueues.cameraQueue.coroutineScope.launch { try { - if (shouldReconfigureDevice) { - configureDevice() - } if (shouldReconfigureSession) { configureSession() } @@ -232,36 +217,30 @@ class CameraView(context: Context) : FrameLayout(context) { } } - /** - * Prepares the hardware Camera Device. (cameraId) - */ - private suspend fun configureDevice() { + private fun configureDevice() { Log.i(TAG, "Configuring Camera Device...") if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { throw CameraPermissionError() } val cameraId = cameraId ?: throw NoCameraDeviceError() - cameraDevice = cameraManager.openCamera(cameraId) { - Log.i(TAG, "Camera Closed!") - cameraSession?.close() - cameraSession = null - cameraDevice = null + cameraDevice = UseCameraDevice(cameraManager, cameraId) { device -> + Log.i(TAG, "Camera Device changed! $device") + reconfigureAll() } } private suspend fun configureSession() { - val cameraDevice = cameraDevice - if (cameraDevice == null) { - Log.w(TAG, "Tried to call configureSession() without a CameraDevice! Returning...") - return - } + val cameraDevice = cameraDevice?.currentValue ?: return + val previewSurface = previewSurface?.currentValue ?: return val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null - val previewSurface = if (previewType == "native") previewView.holder.surface else null + // Close existing session if there is one + cameraSession?.close() + // Start new session cameraSession = CameraSession.createCameraSession( cameraDevice, cameraManager, @@ -280,23 +259,23 @@ class CameraView(context: Context) : FrameLayout(context) { } private fun configureFormat() { - cameraSession?.configureFormat(fps, videoStabilizationMode, hdr, lowLightBoost) + val cameraSession = cameraSession ?: return + + cameraSession.configureFormat(fps, videoStabilizationMode, hdr, lowLightBoost) } private fun updateLifecycle() { - val cameraSession = cameraSession - if (isActive && isAttachedToWindow && cameraSession != null) { - Log.i(TAG, "Starting Camera Session...") + val cameraSession = cameraSession ?: return + + if (isActive && isAttachedToWindow) { cameraSession.startRunning() } else { - Log.i(TAG, "Stopping Camera Session...") - cameraSession?.stopRunning() + cameraSession.stopRunning() } } private fun reconfigureAll() { CameraQueues.cameraQueue.coroutineScope.launch { - configureDevice() configureSession() configureFormat() updateLifecycle() diff --git a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt new file mode 100644 index 0000000000..6c62f9b228 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt @@ -0,0 +1,73 @@ +package com.mrousavy.camera + +import android.annotation.SuppressLint +import android.content.Context +import android.content.res.Resources +import android.graphics.ImageFormat +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraManager +import android.util.Log +import android.util.Size +import android.view.SurfaceView +import com.mrousavy.camera.utils.bigger +import com.mrousavy.camera.utils.smaller +import kotlin.math.max +import kotlin.math.roundToInt + +/** + * A [SurfaceView] that can be adjusted to a specified aspect ratio and + * performs center-crop transformation of input frames. + */ +@SuppressLint("ViewConstructor") +class NativePreviewView(cameraManager: CameraManager, cameraId: String, context: Context): SurfaceView(context) { + private val targetSize: Size + private val aspectRatio: Float + get() = targetSize.width.toFloat() / targetSize.height.toFloat() + + private fun getMaximumPreviewSize(): Size { + // See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap + // According to the Android Developer documentation, PREVIEW streams can have a resolution + // of up to the phone's display's resolution, with a maximum of 1920x1080. + val display1080p = Size(1080, 1920) + val displaySize = Size(Resources.getSystem().displayMetrics.widthPixels, Resources.getSystem().displayMetrics.heightPixels) + val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller + Log.i(TAG, "Phone has a ${displaySize.width} x ${displaySize.height} screen.") + return if (isHighResScreen) display1080p else displaySize + } + + init { + val characteristics = cameraManager.getCameraCharacteristics(cameraId) + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + val previewSize = getMaximumPreviewSize() + val outputSizes = config.getOutputSizes(34 /* TODO: ImageFormat.PRIVATE */).sortedByDescending { it.width * it.height } + targetSize = outputSizes.first { it.bigger <= previewSize.bigger && it.smaller <= previewSize.smaller } + holder.setFixedSize(targetSize.width, targetSize.height) + Log.i(TAG, "Using Preview Size ${targetSize.width} x ${targetSize.height}.") + } + + override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec) + val width = MeasureSpec.getSize(widthMeasureSpec) + val height = MeasureSpec.getSize(heightMeasureSpec) + Log.d(TAG, "onMeasure($width, $height)") + + // Performs center-crop transformation of the camera frames + val newWidth: Int + val newHeight: Int + val actualRatio = if (width > height) aspectRatio else 1f / aspectRatio + if (width < height * actualRatio) { + newHeight = height + newWidth = (height * actualRatio).roundToInt() + } else { + newWidth = width + newHeight = (width / actualRatio).roundToInt() + } + + Log.d(TAG, "Measured dimensions set: $newWidth x $newHeight") + setMeasuredDimension(newWidth, newHeight) + } + + companion object { + private const val TAG = "NativePreviewView" + } +} diff --git a/android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt b/android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt new file mode 100644 index 0000000000..c7ac23be71 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt @@ -0,0 +1,15 @@ +package com.mrousavy.camera.hooks + + +abstract class DataProvider(private val onChange: (value: T?) -> Unit) { + private var value: T? = null + fun update(value: T?) { + if (this.value != value) { + this.value = value + onChange(value) + } + } + + val currentValue: T? + get() = value +} diff --git a/android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt b/android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt new file mode 100644 index 0000000000..fd51b12d15 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt @@ -0,0 +1,74 @@ +package com.mrousavy.camera.hooks + +import android.annotation.SuppressLint +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.util.Log +import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.CameraView +import com.mrousavy.camera.parsers.parseCameraError +import java.io.Closeable + +class UseCameraDevice(private val cameraManager: CameraManager, + val cameraId: String, + onChange: (device: CameraDevice?) -> Unit): Closeable, DataProvider(onChange) { + + private var isOpening = false + private val availabilityCallback = object: CameraManager.AvailabilityCallback() { + override fun onCameraAvailable(id: String) { + super.onCameraAvailable(id) + if (id == cameraId) { + // Our camera is available, try to open it + openCamera() + } + } + + override fun onCameraUnavailable(id: String) { + super.onCameraUnavailable(id) + if (id == cameraId) { + // Our camera is no longer available + update(null) + } + } + } + private val openCameraCallback = object: CameraDevice.StateCallback() { + override fun onOpened(camera: CameraDevice) { + isOpening = false + Log.i(CameraView.TAG, "Successfully opened Camera Device $cameraId!") + update(camera) + } + + override fun onDisconnected(camera: CameraDevice) { + isOpening = false + Log.w(CameraView.TAG, "Camera Device $cameraId has been disconnected! Closing Camera..") + camera.close() + update(null) + } + + override fun onError(camera: CameraDevice, errorCode: Int) { + isOpening = false + Log.e(CameraView.TAG, "Failed to open Camera Device $cameraId! Closing Camera.. " + + "Error: $errorCode (${parseCameraError(errorCode)})") + camera.close() + update(null) + } + } + + init { + cameraManager.registerAvailabilityCallback(availabilityCallback, CameraQueues.cameraQueue.handler) + } + + @SuppressLint("MissingPermission") + fun openCamera() { + if (isOpening || currentValue?.id == cameraId) { + // camera is already opened, no need to re-open. + return + } + isOpening = true + cameraManager.openCamera(cameraId, openCameraCallback, CameraQueues.cameraQueue.handler) + } + + override fun close() { + cameraManager.unregisterAvailabilityCallback(availabilityCallback) + } +} diff --git a/android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt b/android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt new file mode 100644 index 0000000000..021f187f9c --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt @@ -0,0 +1,38 @@ +package com.mrousavy.camera.hooks + +import android.util.Log +import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView +import java.io.Closeable + +class UseSurfaceViewSurface(private val surfaceView: SurfaceView, + onChange: (surface: Surface?) -> Unit): Closeable, DataProvider(onChange) { + companion object { + private const val TAG = "UseSurfaceViewSurface" + } + + private val surfaceCallback = object: SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) { + Log.d(TAG, "Surface Created: ${holder.surface}") + update(holder.surface) + } + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { + Log.d(TAG, "Surface resized: ${holder.surface} $format $width x $height") + } + + override fun surfaceDestroyed(holder: SurfaceHolder) { + Log.d(TAG, "Surface Destroyed: ${holder.surface}") + update(null) + } + } + + init { + surfaceView.holder.addCallback(surfaceCallback) + } + + override fun close() { + surfaceView.holder.removeCallback(surfaceCallback) + } +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt b/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt new file mode 100644 index 0000000000..62a4508c0c --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt @@ -0,0 +1,37 @@ +package com.mrousavy.camera.utils + +import android.util.Size +import android.view.Surface +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min + +fun Array.closestToOrMax(size: Size?): Size { + return if (size != null) { + this.minBy { abs(it.width - size.width) + abs(it.height - size.height) } + } else { + this.maxBy { it.width * it.height } + } +} + +/** + * Rotate by a given Surface Rotation + */ +fun Size.rotated(surfaceRotation: Int): Size { + return when (surfaceRotation) { + Surface.ROTATION_0 -> Size(width, height) + Surface.ROTATION_90 -> Size(height, width) + Surface.ROTATION_180 -> Size(width, height) + Surface.ROTATION_270 -> Size(height, width) + else -> Size(width, height) + } +} + +val Size.bigger: Int + get() = max(width, height) +val Size.smaller: Int + get() = min(width, height) + +operator fun Size.compareTo(other: Size): Int { + return (this.width * this.height).compareTo(other.width * other.height) +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt b/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt deleted file mode 100644 index 3d93f238e5..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/Size+closestToOrMax.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.mrousavy.camera.utils - -import android.util.Size -import kotlin.math.abs - -fun Array.closestToOrMax(size: Size?): Size { - return if (size != null) { - this.minBy { abs(it.width - size.width) + abs(it.height - size.height) } - } else { - this.maxBy { it.width * it.height } - } -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/Size+rotated.kt b/android/src/main/java/com/mrousavy/camera/utils/Size+rotated.kt deleted file mode 100644 index 2c530b40ab..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/Size+rotated.kt +++ /dev/null @@ -1,17 +0,0 @@ -package com.mrousavy.camera.utils - -import android.util.Size -import android.view.Surface - -/** - * Rotate by a given Surface Rotation - */ -fun Size.rotated(surfaceRotation: Int): Size { - return when (surfaceRotation) { - Surface.ROTATION_0 -> Size(width, height) - Surface.ROTATION_90 -> Size(height, width) - Surface.ROTATION_180 -> Size(width, height) - Surface.ROTATION_270 -> Size(height, width) - else -> Size(width, height) - } -} From ea2ef7856a1c39229c3cfcd68ad9c1004d907fd4 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 16:14:35 +0200 Subject: [PATCH 026/180] Properly set up `CameraSession` --- .../java/com/mrousavy/camera/CameraSession.kt | 353 +++++++++++++----- .../java/com/mrousavy/camera/CameraView.kt | 93 ++--- .../com/mrousavy/camera/NativePreviewView.kt | 25 +- .../com/mrousavy/camera/parsers/Size+easy.kt | 17 - .../CameraDevice+createCaptureSession.kt | 14 +- .../mrousavy/camera/utils/Size+Extensions.kt | 7 + ios/CameraView+AVCaptureSession.swift | 2 + ios/CameraView+TakePhoto.swift | 1 + src/CameraProps.ts | 5 + 9 files changed, 343 insertions(+), 174 deletions(-) delete mode 100644 android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index fd81b8113d..4410eecb23 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -1,5 +1,6 @@ package com.mrousavy.camera +import android.annotation.SuppressLint import android.graphics.ImageFormat import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics @@ -14,36 +15,275 @@ import android.util.Range import android.util.Size import android.view.Surface import com.mrousavy.camera.parsers.getVideoStabilizationMode +import com.mrousavy.camera.parsers.parseCameraError import com.mrousavy.camera.utils.OutputType import com.mrousavy.camera.utils.SessionType import com.mrousavy.camera.utils.SurfaceOutput import com.mrousavy.camera.utils.closestToOrMax import com.mrousavy.camera.utils.createCaptureSession +import kotlinx.coroutines.launch import java.io.Closeable import java.lang.IllegalStateException -data class PipelineConfiguration(val enabled: Boolean, - val callback: (image: Image) -> Unit, - val targetSize: Size? = null) -class CameraSession(private val device: CameraDevice, - private val captureSession: CameraCaptureSession, - private val outputs: List): Closeable { - private var captureRequest: CaptureRequest = createCaptureRequestBuilder().build() - private fun createCaptureRequestBuilder(): CaptureRequest.Builder { - val captureRequest = device.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) - outputs.forEach { output -> - if (output.isRepeating) captureRequest.addTarget(output.surface) - } - return captureRequest + +// TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing + +class CameraSession(private val cameraManager: CameraManager, + private val onError: (e: Throwable) -> Unit): Closeable, CameraManager.AvailabilityCallback() { + companion object { + private const val TAG = "CameraSession" + } + /** + * Represents any kind of output for the Camera that delivers Images. Can either be Video or Photo. + */ + data class Output(val enabled: Boolean, + val callback: (image: Image) -> Unit, + val targetSize: Size? = null) + + // setInput(..) + private var cameraId: String? = null + + // setOutputs(..) + private var photoOutput: Output? = null + private var videoOutput: Output? = null + private var previewOutput: Surface? = null + + // setIsActive(..) + private var isActive = false + + // configureFormat(..) + private var fps: Int? = null + private var videoStabilizationMode: String? = null + private var lowLightBoost: Boolean? = null + private var hdr: Boolean? = null + + private val outputs = arrayListOf() + private var captureRequest: CaptureRequest? = null + private var captureSession: CameraCaptureSession? = null + + + init { + cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) + } + + override fun close() { + cameraManager.unregisterAvailabilityCallback(this) + captureSession?.close() + } + + /** + * Set the Camera to be used as an input device. + * Calling this with the same ID twice will not re-open the Camera device. + */ + fun setInputDevice(cameraId: String) { + Log.i(TAG, "Setting Input Device to Camera $cameraId...") + this.cameraId = cameraId + + openCamera(cameraId) + // cameraId changed, prepare outputs. + prepareOutputs() + } + + /** + * Configure the outputs of the Camera. + */ + fun setOutputs(photoOutput: Output? = null, + videoOutput: Output? = null, + previewOutput: Surface? = null) { + this.photoOutput = photoOutput + this.videoOutput = videoOutput + this.previewOutput = previewOutput + // outputs changed, prepare them. + prepareOutputs() } + /** + * Configures various format settings such as FPS, Video Stabilization, HDR or Night Mode. + */ fun configureFormat(fps: Int? = null, videoStabilizationMode: String? = null, hdr: Boolean? = null, lowLightBoost: Boolean? = null) { - val captureRequest = createCaptureRequestBuilder() + this.fps = fps + this.videoStabilizationMode = videoStabilizationMode + this.hdr = hdr + this.lowLightBoost = lowLightBoost + prepareCaptureRequest() + } + + /** + * Starts or stops the Camera. + */ + fun setIsActive(isActive: Boolean) { + if (this.isActive == isActive) { + // We're already active/inactive. + return + } + + this.isActive = isActive + if (isActive) startRunning() + else stopRunning() + } + + override fun onCameraAvailable(cameraId: String) { + super.onCameraAvailable(cameraId) + Log.i(TAG, "Camera became available: $cameraId") + if (cameraId == this.cameraId) { + // The Camera we are trying to use just became available, open it! + openCamera(cameraId) + } + } + + override fun onCameraUnavailable(cameraId: String) { + super.onCameraUnavailable(cameraId) + Log.i(TAG, "Camera became un-available: $cameraId") + } + + @SuppressLint("MissingPermission") + private fun openCamera(cameraId: String) { + if (captureSession?.device?.id == cameraId) { + Log.i(TAG, "Tried to open Camera $cameraId, but we already have a Capture Session running with that Camera. Skipping...") + return + } + + cameraManager.openCamera(cameraId, object: CameraDevice.StateCallback() { + // When Camera is successfully opened (called once) + override fun onOpened(camera: CameraDevice) { + Log.i(TAG, "Camera $cameraId: opened!") + onCameraInitialized(camera) + } + + // When Camera has been disconnected (either called on init, or later) + override fun onDisconnected(camera: CameraDevice) { + Log.i(TAG, "Camera $cameraId: disconnected!") + + onCameraDisconnected() + camera.close() + } + + // When Camera has been encountered an Error (either called on init, or later) + override fun onError(camera: CameraDevice, errorCode: Int) { + val errorString = parseCameraError(errorCode) + onError(CameraCannotBeOpenedError(cameraId, errorString)) + Log.e(TAG, "Camera $cameraId: error! ($errorCode: $errorString)") + + onCameraDisconnected() + camera.close() + } + }, CameraQueues.cameraQueue.handler) + } + + private fun onCameraInitialized(camera: CameraDevice) { + CameraQueues.cameraQueue.coroutineScope.launch { + Log.i(TAG, "Creating CameraCaptureSession for Camera ${camera.id}...") + captureSession?.close() + + captureSession = camera.createCaptureSession( + cameraManager, + SessionType.REGULAR, + outputs, + CameraQueues.cameraQueue + ) + Log.i(TAG, "Successfully created CameraCaptureSession for Camera ${camera.id}!") + + prepareOutputs() + } + } + + private fun onCameraDisconnected() { + captureSession?.close() + captureSession = null + } + + + /** + * Prepares the Image Reader and Surface outputs. + * Call this whenever [cameraId], [photoOutput], [videoOutput], or [previewOutput] changes. + */ + private fun prepareOutputs() { + val cameraId = cameraId ?: return + val videoOutput = videoOutput + val photoOutput = photoOutput + val previewOutput = previewOutput + + Log.i(TAG, "Preparing Outputs for Camera $cameraId...") + + outputs.clear() + + val characteristics = cameraManager.getCameraCharacteristics(cameraId) + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + + if (videoOutput != null) { + // Video or Frame Processor output: High resolution repeating images + val pixelFormat = ImageFormat.YUV_420_888 + val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoOutput.targetSize) + + val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, pixelFormat, 2) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() + if (image == null) { + Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") + return@setOnImageAvailableListener + } + + videoOutput.callback(image) + }, CameraQueues.videoQueue.handler) + + Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") + outputs.add(SurfaceOutput(imageReader.surface, OutputType.VIDEO)) + } + + if (photoOutput != null) { + // Photo output: High quality still images + val pixelFormat = ImageFormat.JPEG + val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoOutput.targetSize) + + val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, pixelFormat, 1) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireLatestImage() + image.use { + Log.d(CameraView.TAG, "Photo captured! ${image.width} x ${image.height}") + photoOutput.callback(image) + } + }, CameraQueues.cameraQueue.handler) + + Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") + outputs.add(SurfaceOutput(imageReader.surface, OutputType.PHOTO)) + } + + if (previewOutput != null) { + // Preview output: Low resolution repeating images + Log.i(CameraView.TAG, "Adding native preview view output.") + outputs.add(SurfaceOutput(previewOutput, OutputType.PREVIEW)) + } + + Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") + } + + + /** + * Prepares the repeating capture request which will be sent to the Camera. + * Call this whenever [captureSession], [fps], [videoStabilizationMode], [hdr], or [lowLightBoost] changes. + */ + private fun prepareCaptureRequest() { + val captureSession = captureSession ?: return + val fps = fps + val videoStabilizationMode = videoStabilizationMode + val hdr = hdr + val lowLightBoost = lowLightBoost + + Log.i(TAG, "Preparing repeating Capture Request...") + + val captureRequest = captureSession.device.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) + outputs.forEach { output -> + if (output.isRepeating) { + Log.i(TAG, "Adding output surface ${output.outputType}..") + captureRequest.addTarget(output.surface) + } + } + if (fps != null) { captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) } @@ -59,10 +299,15 @@ class CameraSession(private val device: CameraDevice, } } this.captureRequest = captureRequest.build() + + // Capture Request changed, restart it + if (isActive) startRunning() } + private fun startRunning() { + val captureSession = captureSession ?: return + val captureRequest = captureRequest ?: return - fun startRunning() { Log.i(TAG, "Starting Camera Session...") try { // Start all repeating requests (Video, Frame Processor, Preview) @@ -72,86 +317,12 @@ class CameraSession(private val device: CameraDevice, } } - fun stopRunning() { + private fun stopRunning() { Log.i(TAG, "Stopping Camera Session...") try { - captureSession.stopRepeating() + captureSession?.stopRepeating() } catch (e: IllegalStateException) { Log.w(TAG, "Failed to stop Camera Session, this session is already closed.") } } - - override fun close() { - stopRunning() - captureSession.close() - } - - companion object { - private const val TAG = "CameraSession" - suspend fun createCameraSession(device: CameraDevice, - cameraManager: CameraManager, - photoPipeline: PipelineConfiguration? = null, - videoPipeline: PipelineConfiguration? = null, - previewSurface: Surface? = null): CameraSession { - val characteristics = cameraManager.getCameraCharacteristics(device.id) - val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - - val outputs = arrayListOf() - - if (videoPipeline != null) { - // Video or Frame Processor output: High resolution repeating images - val pixelFormat = ImageFormat.YUV_420_888 - val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoPipeline.targetSize) - - val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, pixelFormat, 2) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() - if (image == null) { - Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") - return@setOnImageAvailableListener - } - - videoPipeline.callback(image) - }, CameraQueues.videoQueue.handler) - - Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") - val videoOutput = SurfaceOutput(imageReader.surface, OutputType.VIDEO) - outputs.add(videoOutput) - // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing - } - - if (photoPipeline != null) { - // Photo output: High quality still images - val pixelFormat = ImageFormat.JPEG - val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoPipeline.targetSize) - - val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, pixelFormat, 1) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireLatestImage() - image.use { - Log.d(CameraView.TAG, "Photo captured! ${image.width} x ${image.height}") - photoPipeline.callback(image) - } - }, CameraQueues.cameraQueue.handler) - - Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") - val photoOutput = SurfaceOutput(imageReader.surface, OutputType.PHOTO) - outputs.add(photoOutput) - } - - if (previewSurface != null) { - // Preview output: Low resolution repeating images - val previewOutput = SurfaceOutput(previewSurface, OutputType.PREVIEW) - Log.i(CameraView.TAG, "Adding native preview view output.") - outputs.add(previewOutput) - } - - val captureSession = device.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, CameraQueues.cameraQueue) - - Log.i(CameraView.TAG, "Successfully configured Camera Session!") - return CameraSession(device, captureSession, outputs) - } - - } } - diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 4a84c61ddd..a5987df073 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -9,6 +9,7 @@ import android.hardware.camera2.CameraManager import android.util.Log import android.util.Size import android.view.Surface +import android.view.SurfaceHolder import android.view.SurfaceView import android.view.View import android.widget.FrameLayout @@ -60,16 +61,11 @@ class CameraView(context: Context) : FrameLayout(context) { private val propsThatRequirePreviewReconfiguration = arrayListOf("cameraId", "previewType") private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "photo", "video", "enableFrameProcessor") private val propsThatRequireFormatReconfiguration = arrayListOf("fps", "hdr", "videoStabilizationMode", "lowLightBoost") - private val arrayListOfZoom = arrayListOf("zoom") } // react properties // props that require reconfiguring var cameraId: String? = null - set(value) { - field = value - if (value != null) configureDevice() - } var enableDepthData = false var enableHighQualityPhotos: Boolean? = null var enablePortraitEffectsMatteDelivery = false @@ -96,12 +92,11 @@ class CameraView(context: Context) : FrameLayout(context) { private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager // session - private var cameraDevice: UseCameraDevice? = null - private var cameraSession: CameraSession? = null + private var cameraSession: CameraSession private var previewView: View? = null - private var previewSurface: UseSurfaceViewSurface? = null + private var previewSurface: Surface? = null - public var frameProcessor: FrameProcessor? = null + var frameProcessor: FrameProcessor? = null private val inputRotation: Int get() { @@ -130,6 +125,9 @@ class CameraView(context: Context) : FrameLayout(context) { init { this.installHierarchyFitter() setupPreviewView() + cameraSession = CameraSession(cameraManager) { error -> + invokeOnError(error) + } } override fun onConfigurationChanged(newConfig: Configuration?) { @@ -158,12 +156,11 @@ class CameraView(context: Context) : FrameLayout(context) { if (this.previewView is SurfaceView) return removeView(this.previewView) - val previewView = NativePreviewView(cameraManager, cameraId, context) - previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) - previewSurface = UseSurfaceViewSurface(previewView) { - Log.i(TAG, "PreviewView Surface changed!") - reconfigureAll() + val previewView = NativePreviewView(cameraManager, cameraId, context) { surface -> + previewSurface = surface + configureSession() } + previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) addView(previewView) this.previewView = previewView } else { @@ -174,6 +171,7 @@ class CameraView(context: Context) : FrameLayout(context) { fun update(changedProps: ArrayList) { try { val shouldReconfigurePreview = changedProps.containsAny(propsThatRequirePreviewReconfiguration) + val shouldReconfigureDevice = changedProps.contains("cameraId") val shouldReconfigureSession = shouldReconfigurePreview || changedProps.containsAny(propsThatRequireSessionReconfiguration) val shouldReconfigureFormat = shouldReconfigureSession || changedProps.containsAny(propsThatRequireFormatReconfiguration) val shouldReconfigureZoom = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("zoom") @@ -184,21 +182,17 @@ class CameraView(context: Context) : FrameLayout(context) { if (shouldReconfigurePreview) { setupPreviewView() } - CameraQueues.cameraQueue.coroutineScope.launch { - try { - if (shouldReconfigureSession) { - configureSession() - } - if (shouldReconfigureFormat) { - configureFormat() - } - if (shouldCheckActive) { - updateLifecycle() - } - } catch (e: Throwable) { - Log.e(TAG, "Failed to configure Camera!", e) - invokeOnError(e) - } + if (shouldReconfigureDevice) { + configureDevice() + } + if (shouldReconfigureSession) { + configureSession() + } + if (shouldReconfigureFormat) { + configureFormat() + } + if (shouldCheckActive) { + updateLifecycle() } if (shouldReconfigureZoom) { @@ -224,61 +218,34 @@ class CameraView(context: Context) : FrameLayout(context) { } val cameraId = cameraId ?: throw NoCameraDeviceError() - cameraDevice = UseCameraDevice(cameraManager, cameraId) { device -> - Log.i(TAG, "Camera Device changed! $device") - reconfigureAll() - } + cameraSession.setInputDevice(cameraId) } - private suspend fun configureSession() { - val cameraDevice = cameraDevice?.currentValue ?: return - val previewSurface = previewSurface?.currentValue ?: return - + private fun configureSession() { val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null + val previewSurface = if (previewType == "native") previewSurface ?: return else null - // Close existing session if there is one - cameraSession?.close() - // Start new session - cameraSession = CameraSession.createCameraSession( - cameraDevice, - cameraManager, + cameraSession.setOutputs( // Photo Pipeline - PipelineConfiguration(video == true, { + CameraSession.Output(video == true, { Log.i(TAG, "Captured an Image!") }, targetPhotoSize), // Video Pipeline - PipelineConfiguration(photo == true, { image -> + CameraSession.Output(photo == true, { image -> val frame = Frame(image, System.currentTimeMillis(), inputRotation, false) onFrame(frame) }, targetVideoSize), - // Preview Pipeline previewSurface ) } private fun configureFormat() { - val cameraSession = cameraSession ?: return - cameraSession.configureFormat(fps, videoStabilizationMode, hdr, lowLightBoost) } private fun updateLifecycle() { - val cameraSession = cameraSession ?: return - - if (isActive && isAttachedToWindow) { - cameraSession.startRunning() - } else { - cameraSession.stopRunning() - } - } - - private fun reconfigureAll() { - CameraQueues.cameraQueue.coroutineScope.launch { - configureSession() - configureFormat() - updateLifecycle() - } + cameraSession.setIsActive(isActive && isAttachedToWindow) } } diff --git a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt index 6c62f9b228..d88149f1c4 100644 --- a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt +++ b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt @@ -8,6 +8,8 @@ import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.util.Log import android.util.Size +import android.view.Surface +import android.view.SurfaceHolder import android.view.SurfaceView import com.mrousavy.camera.utils.bigger import com.mrousavy.camera.utils.smaller @@ -19,7 +21,10 @@ import kotlin.math.roundToInt * performs center-crop transformation of input frames. */ @SuppressLint("ViewConstructor") -class NativePreviewView(cameraManager: CameraManager, cameraId: String, context: Context): SurfaceView(context) { +class NativePreviewView(cameraManager: CameraManager, + cameraId: String, + context: Context, + private val onSurfaceChanged: (surface: Surface?) -> Unit): SurfaceView(context) { private val targetSize: Size private val aspectRatio: Float get() = targetSize.width.toFloat() / targetSize.height.toFloat() @@ -41,8 +46,24 @@ class NativePreviewView(cameraManager: CameraManager, cameraId: String, context: val previewSize = getMaximumPreviewSize() val outputSizes = config.getOutputSizes(34 /* TODO: ImageFormat.PRIVATE */).sortedByDescending { it.width * it.height } targetSize = outputSizes.first { it.bigger <= previewSize.bigger && it.smaller <= previewSize.smaller } - holder.setFixedSize(targetSize.width, targetSize.height) + Log.i(TAG, "Using Preview Size ${targetSize.width} x ${targetSize.height}.") + holder.setFixedSize(targetSize.width, targetSize.height) + holder.addCallback(object: SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) { + Log.i(TAG, "Surface created! ${holder.surface}") + onSurfaceChanged(holder.surface) + } + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { + Log.i(TAG, "Surface resized! ${holder.surface} ($width x $height in format #$format)") + } + + override fun surfaceDestroyed(holder: SurfaceHolder) { + Log.i(TAG, "Surface destroyed! ${holder.surface}") + onSurfaceChanged(null) + } + }) } override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) { diff --git a/android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt b/android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt deleted file mode 100644 index d10d417bdd..0000000000 --- a/android/src/main/java/com/mrousavy/camera/parsers/Size+easy.kt +++ /dev/null @@ -1,17 +0,0 @@ -package com.mrousavy.camera.parsers - -import android.util.Size -import android.util.SizeF -import kotlin.math.max -import kotlin.math.min - -val Size.bigger: Int - get() = max(this.width, this.height) -val Size.smaller: Int - get() = min(this.width, this.height) - -val SizeF.bigger: Float - get() = max(this.width, this.height) -val SizeF.smaller: Float - get() = min(this.width, this.height) - diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index ea3a6f8077..0b4937b73b 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -80,17 +80,29 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu return false } -suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sessionType: SessionType, outputs: List, queue: CameraQueues.CameraQueue): CameraCaptureSession { +private val TAG = "CreateCaptureSession" + +suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, + sessionType: SessionType, + outputs: List, + queue: CameraQueues.CameraQueue): CameraCaptureSession { return suspendCoroutine { continuation -> val callback = object : CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) { + Log.i(TAG, "Successfully created Capture Session $session (${session.device.id})!") continuation.resume(session) } override fun onConfigureFailed(session: CameraCaptureSession) { + Log.e(TAG, "Failed to create Capture Session $session (${session.device.id})!") continuation.resumeWithException(RuntimeException("Failed to configure the Camera Session!")) } + + override fun onClosed(session: CameraCaptureSession) { + Log.i(TAG, "Capture Session $session (${session.device.id}) has been closed.") + super.onClosed(session) + } } val characteristics = cameraManager.getCameraCharacteristics(this.id) diff --git a/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt b/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt index 62a4508c0c..6d634242c0 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt @@ -1,6 +1,7 @@ package com.mrousavy.camera.utils import android.util.Size +import android.util.SizeF import android.view.Surface import kotlin.math.abs import kotlin.math.max @@ -32,6 +33,12 @@ val Size.bigger: Int val Size.smaller: Int get() = min(width, height) +val SizeF.bigger: Float + get() = max(this.width, this.height) +val SizeF.smaller: Float + get() = min(this.width, this.height) + operator fun Size.compareTo(other: Size): Int { return (this.width * this.height).compareTo(other.width * other.height) } + diff --git a/ios/CameraView+AVCaptureSession.swift b/ios/CameraView+AVCaptureSession.swift index 8dbc37cb3c..afbd744a28 100644 --- a/ios/CameraView+AVCaptureSession.swift +++ b/ios/CameraView+AVCaptureSession.swift @@ -74,8 +74,10 @@ extension CameraView { photoOutput = AVCapturePhotoOutput() if enableHighQualityPhotos?.boolValue == true { + // TODO: In iOS 16 this will be removed in favor of maxPhotoDimensions. photoOutput!.isHighResolutionCaptureEnabled = true if #available(iOS 13.0, *) { + // TODO: Test if this actually does any fusion or if this just calls the captureOutput twice. If the latter, remove it. photoOutput!.isVirtualDeviceConstituentPhotoDeliveryEnabled = photoOutput!.isVirtualDeviceConstituentPhotoDeliverySupported photoOutput!.maxPhotoQualityPrioritization = .quality } else { diff --git a/ios/CameraView+TakePhoto.swift b/ios/CameraView+TakePhoto.swift index df0d02cc45..e2b94c9229 100644 --- a/ios/CameraView+TakePhoto.swift +++ b/ios/CameraView+TakePhoto.swift @@ -44,6 +44,7 @@ extension CameraView { // default, overridable settings if high quality capture was enabled if self.enableHighQualityPhotos?.boolValue == true { + // TODO: On iOS 16+ this will be removed in favor of maxPhotoDimensions. photoSettings.isHighResolutionPhotoEnabled = true if #available(iOS 13.0, *) { photoSettings.photoQualityPrioritization = .quality diff --git a/src/CameraProps.ts b/src/CameraProps.ts index 35583ff226..2115da8d1b 100644 --- a/src/CameraProps.ts +++ b/src/CameraProps.ts @@ -14,6 +14,11 @@ export type FrameProcessor = type: 'skia-frame-processor'; }; +// TODO: Replace `enableHighQualityPhotos: boolean` in favor of `priorization: 'photo' | 'video'` +// TODO: Use RCT_ENUM_PARSER for stuff like previewType, torch, videoStabilizationMode, and orientation +// TODO: Use Photo HostObject for stuff like depthData, portraitEffects, etc. +// TODO: Add RAW capture support + export interface CameraProps extends ViewProps { /** * The Camera Device to use. From 30681a4683ef3217841bfdca42538176abdcab32 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 16:16:29 +0200 Subject: [PATCH 027/180] Delete unused --- .../utils/CameraCharacteristicsUtils.kt | 58 ------------------- .../camera/utils/CameraDeviceDetails.kt | 3 - 2 files changed, 61 deletions(-) delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/CameraCharacteristicsUtils.kt diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraCharacteristicsUtils.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraCharacteristicsUtils.kt deleted file mode 100644 index f9a39f5f67..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraCharacteristicsUtils.kt +++ /dev/null @@ -1,58 +0,0 @@ -package com.mrousavy.camera.utils - -import android.hardware.camera2.CameraCharacteristics -import android.util.Size -import com.facebook.react.bridge.Arguments -import com.facebook.react.bridge.ReadableArray -import com.mrousavy.camera.parsers.bigger -import kotlin.math.PI -import kotlin.math.atan - -// 35mm is 135 film format, a standard in which focal lengths are usually measured -val Size35mm = Size(36, 24) - -/** - * Convert a given array of focal lengths to the corresponding TypeScript union type name. - * - * Possible values for single cameras: - * * `"wide-angle-camera"` - * * `"ultra-wide-angle-camera"` - * * `"telephoto-camera"` - * - * Sources for the focal length categories: - * * [Telephoto Lens (wikipedia)](https://en.wikipedia.org/wiki/Telephoto_lens) - * * [Normal Lens (wikipedia)](https://en.wikipedia.org/wiki/Normal_lens) - * * [Wide-Angle Lens (wikipedia)](https://en.wikipedia.org/wiki/Wide-angle_lens) - * * [Ultra-Wide-Angle Lens (wikipedia)](https://en.wikipedia.org/wiki/Ultra_wide_angle_lens) - */ -fun CameraCharacteristics.getDeviceTypes(): ReadableArray { - // TODO: Check if getDeviceType() works correctly, even for logical multi-cameras - val focalLengths = this.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS)!! - val sensorSize = this.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! - - // To get valid focal length standards we have to upscale to the 35mm measurement (film standard) - val cropFactor = Size35mm.bigger / sensorSize.bigger - - val deviceTypes = Arguments.createArray() - - val containsTelephoto = focalLengths.any { l -> (l * cropFactor) > 35 } // TODO: Telephoto lenses are > 85mm, but we don't have anything between that range.. - // val containsNormalLens = focalLengths.any { l -> (l * cropFactor) > 35 && (l * cropFactor) <= 55 } - val containsWideAngle = focalLengths.any { l -> (l * cropFactor) >= 24 && (l * cropFactor) <= 35 } - val containsUltraWideAngle = focalLengths.any { l -> (l * cropFactor) < 24 } - - if (containsTelephoto) - deviceTypes.pushString("telephoto-camera") - if (containsWideAngle) - deviceTypes.pushString("wide-angle-camera") - if (containsUltraWideAngle) - deviceTypes.pushString("ultra-wide-angle-camera") - - return deviceTypes -} - -fun CameraCharacteristics.getFieldOfView(): Double { - val focalLengths = this.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS)!! - val sensorSize = this.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! - - return 2 * atan(sensorSize.bigger / (focalLengths[0] * 2)) * (180 / PI) -} diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index f1d01d3721..657e423744 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -3,8 +3,6 @@ package com.mrousavy.camera.utils import android.graphics.ImageFormat import android.graphics.PixelFormat import android.hardware.camera2.CameraCharacteristics -import android.hardware.camera2.CameraDevice -import android.hardware.camera2.CameraExtensionCharacteristics import android.hardware.camera2.CameraManager import android.hardware.camera2.CameraMetadata import android.hardware.camera2.params.DynamicRangeProfiles @@ -14,7 +12,6 @@ import android.util.Size import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap -import com.mrousavy.camera.parsers.bigger import com.mrousavy.camera.parsers.parseHardwareLevel import com.mrousavy.camera.parsers.parseImageFormat import com.mrousavy.camera.parsers.parseLensFacing From 71bc242f1a8af9b27447836d95d960c9fa54e36c Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 16:38:59 +0200 Subject: [PATCH 028/180] fix: Fix recreate when outputs change --- .../java/com/mrousavy/camera/CameraSession.kt | 61 +++++++++++++------ .../java/com/mrousavy/camera/CameraView.kt | 2 +- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 4410eecb23..daa311d965 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -25,11 +25,21 @@ import kotlinx.coroutines.launch import java.io.Closeable import java.lang.IllegalStateException - - - // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing +/** + * A Camera Session. + * Flow: + * + * 1. [cameraDevice] gets rebuilt everytime [cameraId] changes + * 2. [outputs] get rebuilt everytime [photoOutput], [videoOutput], [previewOutput] or [cameraDevice] changes. + * 3. [captureSession] gets rebuilt everytime [outputs] changes. + * 4. [startRunning]/[stopRunning] gets called everytime [isActive] or [captureSession] changes. + * + * Examples: + * - Changing [cameraId] causes everything to be rebuilt. + * - Changing [videoOutput] causes all [outputs] to be rebuilt, which later causes the [captureSession] to be rebuilt. + */ class CameraSession(private val cameraManager: CameraManager, private val onError: (e: Throwable) -> Unit): Closeable, CameraManager.AvailabilityCallback() { companion object { @@ -60,6 +70,7 @@ class CameraSession(private val cameraManager: CameraManager, private var hdr: Boolean? = null private val outputs = arrayListOf() + private var cameraDevice: CameraDevice? = null private var captureRequest: CaptureRequest? = null private var captureSession: CameraCaptureSession? = null @@ -117,6 +128,7 @@ class CameraSession(private val cameraManager: CameraManager, * Starts or stops the Camera. */ fun setIsActive(isActive: Boolean) { + Log.i(TAG, "setIsActive($isActive)") if (this.isActive == isActive) { // We're already active/inactive. return @@ -176,23 +188,12 @@ class CameraSession(private val cameraManager: CameraManager, } private fun onCameraInitialized(camera: CameraDevice) { - CameraQueues.cameraQueue.coroutineScope.launch { - Log.i(TAG, "Creating CameraCaptureSession for Camera ${camera.id}...") - captureSession?.close() - - captureSession = camera.createCaptureSession( - cameraManager, - SessionType.REGULAR, - outputs, - CameraQueues.cameraQueue - ) - Log.i(TAG, "Successfully created CameraCaptureSession for Camera ${camera.id}!") - - prepareOutputs() - } + cameraDevice = camera + prepareSession() } private fun onCameraDisconnected() { + cameraDevice = null captureSession?.close() captureSession = null } @@ -260,8 +261,34 @@ class CameraSession(private val cameraManager: CameraManager, } Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") + + // Outputs changed, re-create session + if (cameraDevice != null) prepareSession() } + /** + * Creates the [CameraCaptureSession]. + * Call this whenever [cameraDevice] or [outputs] changes. + */ + private fun prepareSession() { + CameraQueues.cameraQueue.coroutineScope.launch { + val camera = cameraDevice ?: return@launch + if (outputs.isEmpty()) return@launch + + Log.i(TAG, "Creating CameraCaptureSession for Camera ${camera.id}...") + captureSession?.close() + + captureSession = camera.createCaptureSession( + cameraManager, + SessionType.REGULAR, + outputs, + CameraQueues.cameraQueue + ) + Log.i(TAG, "Successfully created CameraCaptureSession for Camera ${camera.id}!") + + prepareCaptureRequest() + } + } /** * Prepares the repeating capture request which will be sent to the Camera. diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index a5987df073..5502794673 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -225,7 +225,7 @@ class CameraView(context: Context) : FrameLayout(context) { val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null - val previewSurface = if (previewType == "native") previewSurface ?: return else null + val previewSurface = if (previewSurface?.isValid == true) previewSurface else null cameraSession.setOutputs( // Photo Pipeline From df1bf40042060d648f313264b42936438f3ecf43 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 16:41:47 +0200 Subject: [PATCH 029/180] Update NativePreviewView.kt --- android/src/main/java/com/mrousavy/camera/NativePreviewView.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt index d88149f1c4..6c0650a09c 100644 --- a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt +++ b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt @@ -33,7 +33,7 @@ class NativePreviewView(cameraManager: CameraManager, // See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap // According to the Android Developer documentation, PREVIEW streams can have a resolution // of up to the phone's display's resolution, with a maximum of 1920x1080. - val display1080p = Size(1080, 1920) + val display1080p = Size(1920, 1080) val displaySize = Size(Resources.getSystem().displayMetrics.widthPixels, Resources.getSystem().displayMetrics.heightPixels) val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller Log.i(TAG, "Phone has a ${displaySize.width} x ${displaySize.height} screen.") From 7317d6cd50aca2e113cf2edf7c2f2dd5d2810a68 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 16:55:56 +0200 Subject: [PATCH 030/180] Use callback for closing --- .../java/com/mrousavy/camera/CameraSession.kt | 50 ++++++++---- .../CameraDevice+createCaptureSession.kt | 81 +++++++------------ 2 files changed, 64 insertions(+), 67 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index daa311d965..b6bd27e5f4 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -160,6 +160,8 @@ class CameraSession(private val cameraManager: CameraManager, return } + Log.i(TAG, "Opening Camera $cameraId...") + cameraManager.openCamera(cameraId, object: CameraDevice.StateCallback() { // When Camera is successfully opened (called once) override fun onOpened(camera: CameraDevice) { @@ -271,23 +273,37 @@ class CameraSession(private val cameraManager: CameraManager, * Call this whenever [cameraDevice] or [outputs] changes. */ private fun prepareSession() { - CameraQueues.cameraQueue.coroutineScope.launch { - val camera = cameraDevice ?: return@launch - if (outputs.isEmpty()) return@launch - - Log.i(TAG, "Creating CameraCaptureSession for Camera ${camera.id}...") - captureSession?.close() - - captureSession = camera.createCaptureSession( - cameraManager, - SessionType.REGULAR, - outputs, - CameraQueues.cameraQueue - ) - Log.i(TAG, "Successfully created CameraCaptureSession for Camera ${camera.id}!") - - prepareCaptureRequest() - } + val camera = cameraDevice ?: return + if (outputs.isEmpty()) return + + Log.i(TAG, "Creating CameraCaptureSession for Camera ${camera.id}...") + captureSession?.close() + captureSession = null + + camera.createCaptureSession( + cameraManager, + SessionType.REGULAR, + outputs, + object: CameraCaptureSession.StateCallback() { + override fun onConfigured(session: CameraCaptureSession) { + Log.d(TAG, "$session Successfully configured Capture Session for Camera ${camera.id}") + captureSession = session + prepareCaptureRequest() + } + + override fun onConfigureFailed(session: CameraCaptureSession) { + Log.d(TAG, "$session Failed to configure Capture Session for Camera ${camera.id}!") + onError(CameraCannotBeOpenedError(camera.id, "session-configuration-failed")) + } + + override fun onClosed(session: CameraCaptureSession) { + super.onClosed(session) + Log.d(TAG, "$session Capture Session for Camera ${camera.id} closed!") + captureSession = null + } + }, + CameraQueues.cameraQueue + ) } /** diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 0b4937b73b..8b6a832f81 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -80,60 +80,41 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu return false } -private val TAG = "CreateCaptureSession" - -suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, - sessionType: SessionType, - outputs: List, - queue: CameraQueues.CameraQueue): CameraCaptureSession { - return suspendCoroutine { continuation -> - - val callback = object : CameraCaptureSession.StateCallback() { - override fun onConfigured(session: CameraCaptureSession) { - Log.i(TAG, "Successfully created Capture Session $session (${session.device.id})!") - continuation.resume(session) - } - - override fun onConfigureFailed(session: CameraCaptureSession) { - Log.e(TAG, "Failed to create Capture Session $session (${session.device.id})!") - continuation.resumeWithException(RuntimeException("Failed to configure the Camera Session!")) - } - - override fun onClosed(session: CameraCaptureSession) { - Log.i(TAG, "Capture Session $session (${session.device.id}) has been closed.") - super.onClosed(session) +fun CameraDevice.createCaptureSession(cameraManager: CameraManager, + sessionType: SessionType, + outputs: List, + callback: CameraCaptureSession.StateCallback, + queue: CameraQueues.CameraQueue) { + val characteristics = cameraManager.getCameraCharacteristics(this.id) + val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! + Log.i(CameraView.TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + // API >= 24 + val outputConfigurations = outputs.map { + val result = OutputConfiguration(it.surface) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + if (it.isMirrored) result.mirrorMode = OutputConfiguration.MIRROR_MODE_H + if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile + if (supportsOutputType(characteristics, it.outputType)) { + result.streamUseCase = it.outputType.toOutputType() + Log.i(CameraView.TAG, "Using optimized stream use case \"${it.outputType.name}\" (${result.streamUseCase})..") + } } + return@map result } - val characteristics = cameraManager.getCameraCharacteristics(this.id) - val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! - Log.i(CameraView.TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - val outputConfigurations = outputs.map { - val result = OutputConfiguration(it.surface) - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - if (it.isMirrored) result.mirrorMode = OutputConfiguration.MIRROR_MODE_H - if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile - if (supportsOutputType(characteristics, it.outputType)) { - result.streamUseCase = it.outputType.toOutputType() - Log.i(CameraView.TAG, "Using optimized stream use case \"${it.outputType.name}\" (${result.streamUseCase})..") - } - } - return@map result - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - // API >=28 - val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) - this.createCaptureSession(config) - } else { - // API >=24 - this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) - } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + // API >=28 + val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) + this.createCaptureSession(config) } else { - // API <24 - this.createCaptureSession(outputs.map { it.surface }, callback, queue.handler) + // API >=24 + this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) } + } else { + // API <24 + this.createCaptureSession(outputs.map { it.surface }, callback, queue.handler) } } From 8b8ba140c047fa3d2b8829cffeb57a3334a5cd36 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 17:26:30 +0200 Subject: [PATCH 031/180] Catch CameraAccessException --- .../java/com/mrousavy/camera/CameraSession.kt | 63 +++++++++++-------- .../java/com/mrousavy/camera/CameraView.kt | 4 +- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index b6bd27e5f4..68ec7574d6 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -2,6 +2,7 @@ package com.mrousavy.camera import android.annotation.SuppressLint import android.graphics.ImageFormat +import android.hardware.camera2.CameraAccessException import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice @@ -280,30 +281,39 @@ class CameraSession(private val cameraManager: CameraManager, captureSession?.close() captureSession = null - camera.createCaptureSession( - cameraManager, - SessionType.REGULAR, - outputs, - object: CameraCaptureSession.StateCallback() { - override fun onConfigured(session: CameraCaptureSession) { - Log.d(TAG, "$session Successfully configured Capture Session for Camera ${camera.id}") - captureSession = session - prepareCaptureRequest() - } - - override fun onConfigureFailed(session: CameraCaptureSession) { - Log.d(TAG, "$session Failed to configure Capture Session for Camera ${camera.id}!") - onError(CameraCannotBeOpenedError(camera.id, "session-configuration-failed")) - } - - override fun onClosed(session: CameraCaptureSession) { - super.onClosed(session) - Log.d(TAG, "$session Capture Session for Camera ${camera.id} closed!") - captureSession = null - } - }, - CameraQueues.cameraQueue - ) + try { + camera.createCaptureSession( + cameraManager, + SessionType.REGULAR, + outputs, + object : CameraCaptureSession.StateCallback() { + override fun onConfigured(session: CameraCaptureSession) { + Log.d(TAG, "$session Successfully configured Capture Session for Camera ${camera.id}") + captureSession = session + prepareCaptureRequest() + } + + override fun onConfigureFailed(session: CameraCaptureSession) { + Log.d(TAG, "$session Failed to configure Capture Session for Camera ${camera.id}!") + onError(CameraCannotBeOpenedError(camera.id, "session-configuration-failed")) + } + + override fun onClosed(session: CameraCaptureSession) { + super.onClosed(session) + Log.d(TAG, "$session Capture Session for Camera ${camera.id} closed!") + try { + session.close() + } catch (_: Throwable) { + } + captureSession = null + } + }, + CameraQueues.cameraQueue + ) + } catch (e: CameraAccessException) { + Log.e(TAG, "Camera Access Exception!", e) + onError(CameraCannotBeOpenedError(camera.id, "camera-not-connected-anymore")) + } } /** @@ -355,6 +365,7 @@ class CameraSession(private val cameraManager: CameraManager, try { // Start all repeating requests (Video, Frame Processor, Preview) captureSession.setRepeatingRequest(captureRequest, null, null) + Log.i(TAG, "Camera Session started!") } catch (e: IllegalStateException) { Log.w(TAG, "Failed to start Camera Session, this session is already closed.") } @@ -363,7 +374,9 @@ class CameraSession(private val cameraManager: CameraManager, private fun stopRunning() { Log.i(TAG, "Stopping Camera Session...") try { - captureSession?.stopRepeating() + val captureSession = captureSession ?: return + captureSession.stopRepeating() + Log.i(TAG, "Camera Session stopped!") } catch (e: IllegalStateException) { Log.w(TAG, "Failed to stop Camera Session, this session is already closed.") } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 5502794673..e0b2d0029c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -225,7 +225,9 @@ class CameraView(context: Context) : FrameLayout(context) { val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null - val previewSurface = if (previewSurface?.isValid == true) previewSurface else null + val previewSurface = previewSurface ?: return + + if (!previewSurface.isValid) return cameraSession.setOutputs( // Photo Pipeline From 1e212d069130a062e05309a6e49eb57cb9150e54 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 17:38:08 +0200 Subject: [PATCH 032/180] Finally got it stable --- .../java/com/mrousavy/camera/CameraSession.kt | 83 ++++++++----------- 1 file changed, 33 insertions(+), 50 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 68ec7574d6..b541df4006 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -22,7 +22,6 @@ import com.mrousavy.camera.utils.SessionType import com.mrousavy.camera.utils.SurfaceOutput import com.mrousavy.camera.utils.closestToOrMax import com.mrousavy.camera.utils.createCaptureSession -import kotlinx.coroutines.launch import java.io.Closeable import java.lang.IllegalStateException @@ -72,9 +71,8 @@ class CameraSession(private val cameraManager: CameraManager, private val outputs = arrayListOf() private var cameraDevice: CameraDevice? = null - private var captureRequest: CaptureRequest? = null private var captureSession: CameraCaptureSession? = null - + private var cameraIdCurrentlyOpening: String? = null init { cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) @@ -122,7 +120,6 @@ class CameraSession(private val cameraManager: CameraManager, this.videoStabilizationMode = videoStabilizationMode this.hdr = hdr this.lowLightBoost = lowLightBoost - prepareCaptureRequest() } /** @@ -156,6 +153,9 @@ class CameraSession(private val cameraManager: CameraManager, @SuppressLint("MissingPermission") private fun openCamera(cameraId: String) { + if (cameraIdCurrentlyOpening == cameraId) return + cameraIdCurrentlyOpening = cameraId + if (captureSession?.device?.id == cameraId) { Log.i(TAG, "Tried to open Camera $cameraId, but we already have a Capture Session running with that Camera. Skipping...") return @@ -207,6 +207,8 @@ class CameraSession(private val cameraManager: CameraManager, * Call this whenever [cameraId], [photoOutput], [videoOutput], or [previewOutput] changes. */ private fun prepareOutputs() { + outputs.clear() + val cameraId = cameraId ?: return val videoOutput = videoOutput val photoOutput = photoOutput @@ -214,8 +216,6 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Preparing Outputs for Camera $cameraId...") - outputs.clear() - val characteristics = cameraManager.getCameraCharacteristics(cameraId) val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! @@ -290,7 +290,7 @@ class CameraSession(private val cameraManager: CameraManager, override fun onConfigured(session: CameraCaptureSession) { Log.d(TAG, "$session Successfully configured Capture Session for Camera ${camera.id}") captureSession = session - prepareCaptureRequest() + if (isActive) startRunning() } override fun onConfigureFailed(session: CameraCaptureSession) { @@ -316,55 +316,38 @@ class CameraSession(private val cameraManager: CameraManager, } } - /** - * Prepares the repeating capture request which will be sent to the Camera. - * Call this whenever [captureSession], [fps], [videoStabilizationMode], [hdr], or [lowLightBoost] changes. - */ - private fun prepareCaptureRequest() { - val captureSession = captureSession ?: return - val fps = fps - val videoStabilizationMode = videoStabilizationMode - val hdr = hdr - val lowLightBoost = lowLightBoost - - Log.i(TAG, "Preparing repeating Capture Request...") - - val captureRequest = captureSession.device.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) - outputs.forEach { output -> - if (output.isRepeating) { - Log.i(TAG, "Adding output surface ${output.outputType}..") - captureRequest.addTarget(output.surface) - } - } - - if (fps != null) { - captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) - } - if (videoStabilizationMode != null) { - captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, getVideoStabilizationMode(videoStabilizationMode)) - } - if (lowLightBoost == true) { - captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) - } - if (hdr == true) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP_MR1) { - captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR) - } - } - this.captureRequest = captureRequest.build() - - // Capture Request changed, restart it - if (isActive) startRunning() - } - private fun startRunning() { val captureSession = captureSession ?: return - val captureRequest = captureRequest ?: return Log.i(TAG, "Starting Camera Session...") try { + Log.i(TAG, "Preparing repeating Capture Request...") + + val captureRequest = captureSession.device.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) + outputs.forEach { output -> + if (output.isRepeating) { + Log.i(TAG, "Adding output surface ${output.outputType}..") + captureRequest.addTarget(output.surface) + } + } + + fps?.let { fps -> + captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) + } + videoStabilizationMode?.let { videoStabilizationMode -> + captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, getVideoStabilizationMode(videoStabilizationMode)) + } + if (lowLightBoost == true) { + captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) + } + if (hdr == true) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP_MR1) { + captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR) + } + } + // Start all repeating requests (Video, Frame Processor, Preview) - captureSession.setRepeatingRequest(captureRequest, null, null) + captureSession.setRepeatingRequest(captureRequest.build(), null, null) Log.i(TAG, "Camera Session started!") } catch (e: IllegalStateException) { Log.w(TAG, "Failed to start Camera Session, this session is already closed.") From 22d07dd9bbeeb15137356c664b2c110d4a91f472 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 18:05:00 +0200 Subject: [PATCH 033/180] Remove isMirrored --- .../CameraDevice+createCaptureSession.kt | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 8b6a832f81..52ef822e64 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -54,7 +54,6 @@ enum class OutputType { data class SurfaceOutput(val surface: Surface, val outputType: OutputType, - val isMirrored: Boolean = false, val dynamicRangeProfile: Long? = null) { val isRepeating: Boolean get() = outputType == OutputType.VIDEO || outputType == OutputType.PREVIEW || outputType == OutputType.VIDEO_AND_PREVIEW @@ -80,6 +79,8 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu return false } +private const val TAG = "CreateCaptureSession" + fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sessionType: SessionType, outputs: List, @@ -87,22 +88,29 @@ fun CameraDevice.createCaptureSession(cameraManager: CameraManager, queue: CameraQueues.CameraQueue) { val characteristics = cameraManager.getCameraCharacteristics(this.id) val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! - Log.i(CameraView.TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") + Log.i(TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { // API >= 24 - val outputConfigurations = outputs.map { - val result = OutputConfiguration(it.surface) + val outputConfigurations = arrayListOf() + for (output in outputs) { + if (!output.surface.isValid) { + Log.w(TAG, "Tried to add ${output.outputType} output, but Surface was invalid! Skipping this output..") + continue + } + val result = OutputConfiguration(output.surface) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - if (it.isMirrored) result.mirrorMode = OutputConfiguration.MIRROR_MODE_H - if (it.dynamicRangeProfile != null) result.dynamicRangeProfile = it.dynamicRangeProfile - if (supportsOutputType(characteristics, it.outputType)) { - result.streamUseCase = it.outputType.toOutputType() - Log.i(CameraView.TAG, "Using optimized stream use case \"${it.outputType.name}\" (${result.streamUseCase})..") + if (output.dynamicRangeProfile != null) { + result.dynamicRangeProfile = output.dynamicRangeProfile + Log.i(TAG, "Using dynamic range profile ${result.dynamicRangeProfile} for ${output.outputType} output.") + } + if (supportsOutputType(characteristics, output.outputType)) { + result.streamUseCase = output.outputType.toOutputType() + Log.i(TAG, "Using optimized stream use case ${result.streamUseCase} for ${output.outputType} output.") } } - return@map result + outputConfigurations.add(result) } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { From db7f778e28a1f040db147f193f5f7774c87408bd Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 20:52:58 +0200 Subject: [PATCH 034/180] Implement `takePhoto()` --- .../java/com/mrousavy/camera/CameraSession.kt | 105 +++++++++++++----- .../mrousavy/camera/CameraView+TakePhoto.kt | 84 +++++++++++++- .../java/com/mrousavy/camera/CameraView.kt | 23 ++-- .../main/java/com/mrousavy/camera/Errors.kt | 4 +- .../parsers/VideoStabilizationMode+String.kt | 24 +++- .../utils/CameraCaptureSession+capture.kt | 42 +++++++ .../CameraDevice+createPhotoCaptureRequest.kt | 88 +++++++++++++++ .../com/mrousavy/camera/utils/ExifUtils.kt | 20 ++++ .../camera/utils/PhotoOutputSynchronizer.kt | 26 +++++ example/android/app/build.gradle | 1 - src/CameraDevice.ts | 10 +- src/PhotoFile.ts | 9 +- 12 files changed, 382 insertions(+), 54 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/utils/CameraCaptureSession+capture.kt create mode 100644 android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt create mode 100644 android/src/main/java/com/mrousavy/camera/utils/ExifUtils.kt create mode 100644 android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index b541df4006..0b7839a306 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -8,6 +8,8 @@ import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult import android.media.Image import android.media.ImageReader import android.os.Build @@ -17,13 +19,19 @@ import android.util.Size import android.view.Surface import com.mrousavy.camera.parsers.getVideoStabilizationMode import com.mrousavy.camera.parsers.parseCameraError +import com.mrousavy.camera.utils.ExifUtils +import com.mrousavy.camera.utils.FlashMode import com.mrousavy.camera.utils.OutputType +import com.mrousavy.camera.utils.PhotoOutputSynchronizer +import com.mrousavy.camera.utils.QualityPrioritization import com.mrousavy.camera.utils.SessionType import com.mrousavy.camera.utils.SurfaceOutput +import com.mrousavy.camera.utils.capture import com.mrousavy.camera.utils.closestToOrMax import com.mrousavy.camera.utils.createCaptureSession +import com.mrousavy.camera.utils.createPhotoCaptureRequest import java.io.Closeable -import java.lang.IllegalStateException +import java.util.concurrent.CancellationException // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing @@ -41,24 +49,33 @@ import java.lang.IllegalStateException * - Changing [videoOutput] causes all [outputs] to be rebuilt, which later causes the [captureSession] to be rebuilt. */ class CameraSession(private val cameraManager: CameraManager, - private val onError: (e: Throwable) -> Unit): Closeable, CameraManager.AvailabilityCallback() { + private val onInitialized: () -> Unit, + private val onError: (e: Throwable) -> Unit): Closeable, CameraManager.AvailabilityCallback() { companion object { private const val TAG = "CameraSession" + private const val PHOTO_OUTPUT_BUFFER_SIZE = 3 + private const val VIDEO_OUTPUT_BUFFER_SIZE = 2 } - /** - * Represents any kind of output for the Camera that delivers Images. Can either be Video or Photo. - */ - data class Output(val enabled: Boolean, - val callback: (image: Image) -> Unit, - val targetSize: Size? = null) + data class VideoOutput(val enabled: Boolean, + val callback: (image: Image) -> Unit, + val targetSize: Size? = null) + data class PhotoOutput(val enabled: Boolean, + val targetSize: Size? = null) + data class PreviewOutput(val enabled: Boolean, + val surface: Surface) + + data class CapturedPhoto(val image: Image, + val metadata: TotalCaptureResult, + val orientation: Int, + val format: Int) // setInput(..) private var cameraId: String? = null // setOutputs(..) - private var photoOutput: Output? = null - private var videoOutput: Output? = null - private var previewOutput: Surface? = null + private var photoOutput: PhotoOutput? = null + private var videoOutput: VideoOutput? = null + private var previewOutput: PreviewOutput? = null // setIsActive(..) private var isActive = false @@ -73,6 +90,7 @@ class CameraSession(private val cameraManager: CameraManager, private var cameraDevice: CameraDevice? = null private var captureSession: CameraCaptureSession? = null private var cameraIdCurrentlyOpening: String? = null + private val photoOutputSynchronizer = PhotoOutputSynchronizer() init { cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) @@ -99,9 +117,9 @@ class CameraSession(private val cameraManager: CameraManager, /** * Configure the outputs of the Camera. */ - fun setOutputs(photoOutput: Output? = null, - videoOutput: Output? = null, - previewOutput: Surface? = null) { + fun setOutputs(photoOutput: PhotoOutput? = null, + videoOutput: VideoOutput? = null, + previewOutput: PreviewOutput? = null) { this.photoOutput = photoOutput this.videoOutput = videoOutput this.previewOutput = previewOutput @@ -137,6 +155,35 @@ class CameraSession(private val cameraManager: CameraManager, else stopRunning() } + suspend fun takePhoto(qualityPrioritization: QualityPrioritization, + flashMode: FlashMode, + enableRedEyeReduction: Boolean, + enableAutoStabilization: Boolean): CapturedPhoto { + val captureSession = captureSession ?: throw CameraNotReadyError() + + val captureRequest = captureSession.device.createPhotoCaptureRequest(cameraManager, + qualityPrioritization, + flashMode, + enableRedEyeReduction, + enableAutoStabilization) + val result = captureSession.capture(captureRequest) + val timestamp = result[CaptureResult.SENSOR_TIMESTAMP]!! + try { + val image = photoOutputSynchronizer[timestamp].await() + // TODO: Correctly get rotationDegrees and isMirrored + val rotation = ExifUtils.computeExifOrientation(0, false) + + return CapturedPhoto(image, result, rotation, image.format) + } catch (e: CancellationException) { + throw CaptureAbortedError(false) + } + } + + private fun onPhotoCaptured(image: Image) { + Log.i(CameraView.TAG, "Photo captured! ${image.width} x ${image.height}") + photoOutputSynchronizer.set(image.timestamp, image) + } + override fun onCameraAvailable(cameraId: String) { super.onCameraAvailable(cameraId) Log.i(TAG, "Camera became available: $cameraId") @@ -219,12 +266,15 @@ class CameraSession(private val cameraManager: CameraManager, val characteristics = cameraManager.getCameraCharacteristics(cameraId) val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - if (videoOutput != null) { + if (videoOutput != null && videoOutput.enabled) { // Video or Frame Processor output: High resolution repeating images val pixelFormat = ImageFormat.YUV_420_888 val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoOutput.targetSize) - val imageReader = ImageReader.newInstance(videoSize.width, videoSize.height, pixelFormat, 2) + val imageReader = ImageReader.newInstance(videoSize.width, + videoSize.height, + pixelFormat, + VIDEO_OUTPUT_BUFFER_SIZE) imageReader.setOnImageAvailableListener({ reader -> val image = reader.acquireNextImage() if (image == null) { @@ -239,28 +289,28 @@ class CameraSession(private val cameraManager: CameraManager, outputs.add(SurfaceOutput(imageReader.surface, OutputType.VIDEO)) } - if (photoOutput != null) { + if (photoOutput != null && photoOutput.enabled) { // Photo output: High quality still images val pixelFormat = ImageFormat.JPEG val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoOutput.targetSize) - val imageReader = ImageReader.newInstance(photoSize.width, photoSize.height, pixelFormat, 1) + val imageReader = ImageReader.newInstance(photoSize.width, + photoSize.height, + pixelFormat, + PHOTO_OUTPUT_BUFFER_SIZE) imageReader.setOnImageAvailableListener({ reader -> val image = reader.acquireLatestImage() - image.use { - Log.d(CameraView.TAG, "Photo captured! ${image.width} x ${image.height}") - photoOutput.callback(image) - } + onPhotoCaptured(image) }, CameraQueues.cameraQueue.handler) Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") outputs.add(SurfaceOutput(imageReader.surface, OutputType.PHOTO)) } - if (previewOutput != null) { + if (previewOutput != null && previewOutput.enabled) { // Preview output: Low resolution repeating images Log.i(CameraView.TAG, "Adding native preview view output.") - outputs.add(SurfaceOutput(previewOutput, OutputType.PREVIEW)) + outputs.add(SurfaceOutput(previewOutput.surface, OutputType.PREVIEW)) } Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") @@ -280,6 +330,7 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Creating CameraCaptureSession for Camera ${camera.id}...") captureSession?.close() captureSession = null + photoOutputSynchronizer.clear() try { camera.createCaptureSession( @@ -335,7 +386,9 @@ class CameraSession(private val cameraManager: CameraManager, captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) } videoStabilizationMode?.let { videoStabilizationMode -> - captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, getVideoStabilizationMode(videoStabilizationMode)) + val stabilizationMode = getVideoStabilizationMode(videoStabilizationMode) + captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, stabilizationMode.digitalMode) + captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, stabilizationMode.opticalMode) } if (lowLightBoost == true) { captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) @@ -349,6 +402,8 @@ class CameraSession(private val cameraManager: CameraManager, // Start all repeating requests (Video, Frame Processor, Preview) captureSession.setRepeatingRequest(captureRequest.build(), null, null) Log.i(TAG, "Camera Session started!") + + onInitialized() } catch (e: IllegalStateException) { Log.w(TAG, "Failed to start Camera Session, this session is already closed.") } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index e40e08aad0..5ac65a32c5 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -1,17 +1,99 @@ package com.mrousavy.camera import android.annotation.SuppressLint +import android.content.Context +import android.graphics.ImageFormat import android.hardware.camera2.* import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap +import com.mrousavy.camera.parsers.getVideoStabilizationMode import com.mrousavy.camera.utils.* import kotlinx.coroutines.* +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Files +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale @SuppressLint("UnsafeOptInUsageError") -suspend fun CameraView.takePhoto(options: ReadableMap): WritableMap = coroutineScope { +suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = coroutineScope { // TODO: takePhoto() + val options = optionsMap.toHashMap() + + val qualityPrioritization = options["qualityPrioritization"] as? String + val flash = options["flash"] as? String + val enableAutoRedEyeReduction = options["enableAutoRedEyeReduction"] == true + val enableAutoStabilization = options["enableAutoStabilization"] == true + val skipMetadata = options["skipMetadata"] == true + + val flashMode = when (flash) { + "off" -> FlashMode.OFF + "on" -> FlashMode.ON + "auto" -> FlashMode.AUTO + else -> FlashMode.AUTO + } + val qualityPrioritizationMode = when (qualityPrioritization) { + "speed" -> QualityPrioritization.SPEED + "balanced" -> QualityPrioritization.BALANCED + "quality" -> QualityPrioritization.QUALITY + else -> QualityPrioritization.BALANCED + } + + val photo = cameraSession.takePhoto(qualityPrioritizationMode, + flashMode, + enableAutoRedEyeReduction, + enableAutoStabilization) + + val cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraId!!) + + val path = savePhotoToFile(context, cameraCharacteristics, photo) val map = Arguments.createMap() + map.putString("path", path) + map.putInt("width", photo.image.width) + map.putInt("height", photo.image.height) + map.putBoolean("isRawPhoto", photo.format == ImageFormat.RAW_SENSOR) + + // TODO: Add metadata prop to resulting photo + return@coroutineScope map } + +private suspend fun savePhotoToFile(context: Context, + cameraCharacteristics: CameraCharacteristics, + photo: CameraSession.CapturedPhoto): String { + return withContext(Dispatchers.IO) { + when (photo.format) { + // When the format is JPEG or DEPTH JPEG we can simply save the bytes as-is + ImageFormat.JPEG, ImageFormat.DEPTH_JPEG -> { + val buffer = photo.image.planes[0].buffer + val bytes = ByteArray(buffer.remaining()).apply { buffer.get(this) } + val file = createFile(context, "jpg") + FileOutputStream(file).use { stream -> + stream.write(bytes) + } + return@withContext file.absolutePath + } + + // When the format is RAW we use the DngCreator utility library + ImageFormat.RAW_SENSOR -> { + val dngCreator = DngCreator(cameraCharacteristics, photo.metadata) + val file = createFile(context, "dng") + FileOutputStream(file).use { stream -> + dngCreator.writeImage(stream, photo.image) + } + return@withContext file.absolutePath + } + + else -> { + throw Error("Failed to save Photo to file, image format is not supported! ${photo.format}") + } + } + } +} + +private fun createFile(context: Context, extension: String): File { + return File.createTempFile("mrousavy", extension, context.cacheDir).apply { deleteOnExit() } +} diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index e0b2d0029c..71d9c27bae 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -89,10 +89,10 @@ class CameraView(context: Context) : FrameLayout(context) { // private properties private var isMounted = false - private val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager + internal val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager // session - private var cameraSession: CameraSession + internal val cameraSession: CameraSession private var previewView: View? = null private var previewSurface: Surface? = null @@ -125,9 +125,11 @@ class CameraView(context: Context) : FrameLayout(context) { init { this.installHierarchyFitter() setupPreviewView() - cameraSession = CameraSession(cameraManager) { error -> + cameraSession = CameraSession(cameraManager, { + invokeOnInitialized() + }, { error -> invokeOnError(error) - } + }) } override fun onConfigurationChanged(newConfig: Configuration?) { @@ -230,16 +232,15 @@ class CameraView(context: Context) : FrameLayout(context) { if (!previewSurface.isValid) return cameraSession.setOutputs( - // Photo Pipeline - CameraSession.Output(video == true, { - Log.i(TAG, "Captured an Image!") - }, targetPhotoSize), - // Video Pipeline - CameraSession.Output(photo == true, { image -> + // Photo Output + CameraSession.PhotoOutput(photo == true, targetPhotoSize), + // Video Output + CameraSession.VideoOutput(video == true, { image -> val frame = Frame(image, System.currentTimeMillis(), inputRotation, false) onFrame(frame) }, targetVideoSize), - previewSurface + // Preview Output + CameraSession.PreviewOutput(true, previewSurface) ) } diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 74783ebd77..0a9d62f513 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -56,8 +56,8 @@ class CameraDisconnectedError(cameraId: String) : CameraError("session", "camera class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.") - -class InvalidFormatError(format: Int) : CameraError("capture", "invalid-photo-format", "The Photo has an invalid format! Expected ${ImageFormat.YUV_420_888}, actual: $format") +class CaptureAbortedError(wasImageCaptured: Boolean) : CameraError("capture", "aborted", "The image capture was aborted! Was Image captured: $wasImageCaptured") +class UnknownCaptureError(wasImageCaptured: Boolean) : CameraError("capture", "unknown", "An unknown error occurred while trying to capture an Image! Was Image captured: $wasImageCaptured") class VideoEncoderError(cause: Throwable?) : CameraError("capture", "encoder-error", "The recording failed while encoding.\n" + "This error may be generated when the video or audio codec encounters an error during encoding. " + diff --git a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt index 21563084a9..7dbaaea665 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt @@ -1,8 +1,12 @@ package com.mrousavy.camera.parsers import android.hardware.camera2.CameraMetadata.* +import android.os.Build -fun parseVideoStabilizationMode(stabiliazionMode: Int): String { +data class VideoStabilizationMode(val digitalMode: Int, + val opticalMode: Int) + +fun parseDigitalVideoStabilizationMode(stabiliazionMode: Int): String { return when (stabiliazionMode) { CONTROL_VIDEO_STABILIZATION_MODE_OFF -> "off" CONTROL_VIDEO_STABILIZATION_MODE_ON -> "standard" @@ -10,12 +14,20 @@ fun parseVideoStabilizationMode(stabiliazionMode: Int): String { else -> "off" } } +fun parseOpticalVideoStabilizationMode(stabiliazionMode: Int): String { + return when (stabiliazionMode) { + LENS_OPTICAL_STABILIZATION_MODE_OFF -> "off" + LENS_OPTICAL_STABILIZATION_MODE_ON -> "cinematic-extended" + else -> "off" + } +} -fun getVideoStabilizationMode(stabiliazionMode: String): Int { +fun getVideoStabilizationMode(stabiliazionMode: String): VideoStabilizationMode { return when (stabiliazionMode) { - "off" -> CONTROL_VIDEO_STABILIZATION_MODE_OFF - "standard" -> CONTROL_VIDEO_STABILIZATION_MODE_ON - "cinematic" -> 2 /* TODO: CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION */ - else -> CONTROL_VIDEO_STABILIZATION_MODE_OFF + "off" -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_OFF, LENS_OPTICAL_STABILIZATION_MODE_OFF) + "standard" -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_ON, LENS_OPTICAL_STABILIZATION_MODE_OFF) + "cinematic" -> VideoStabilizationMode(2 /* CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION */, LENS_OPTICAL_STABILIZATION_MODE_OFF) + "cinematic-extended" -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_OFF, LENS_OPTICAL_STABILIZATION_MODE_ON) + else -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_OFF, LENS_OPTICAL_STABILIZATION_MODE_OFF) } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraCaptureSession+capture.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraCaptureSession+capture.kt new file mode 100644 index 0000000000..7d407717e7 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraCaptureSession+capture.kt @@ -0,0 +1,42 @@ +package com.mrousavy.camera.utils + +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CaptureFailure +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.TotalCaptureResult +import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.CaptureAbortedError +import com.mrousavy.camera.UnknownCaptureError +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine + +suspend fun CameraCaptureSession.capture(captureRequest: CaptureRequest): TotalCaptureResult { + return suspendCoroutine { continuation -> + this.capture(captureRequest, object: CameraCaptureSession.CaptureCallback() { + override fun onCaptureCompleted( + session: CameraCaptureSession, + request: CaptureRequest, + result: TotalCaptureResult + ) { + super.onCaptureCompleted(session, request, result) + continuation.resume(result) + } + + override fun onCaptureFailed( + session: CameraCaptureSession, + request: CaptureRequest, + failure: CaptureFailure + ) { + super.onCaptureFailed(session, request, failure) + val wasImageCaptured = failure.wasImageCaptured() + val error = when (failure.reason) { + CaptureFailure.REASON_ERROR -> UnknownCaptureError(wasImageCaptured) + CaptureFailure.REASON_FLUSHED -> CaptureAbortedError(wasImageCaptured) + else -> UnknownCaptureError(wasImageCaptured) + } + continuation.resumeWithException(error) + } + }, CameraQueues.cameraQueue.handler) + } +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt new file mode 100644 index 0000000000..c9c876d553 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt @@ -0,0 +1,88 @@ +package com.mrousavy.camera.utils + +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.hardware.camera2.CaptureRequest +import android.os.Build + + +enum class FlashMode { OFF, ON, AUTO } + +enum class QualityPrioritization { SPEED, BALANCED, QUALITY } + +private fun supportsDeviceZsl(capabilities: IntArray): Boolean { + if (capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_PRIVATE_REPROCESSING)) return true + if (capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_YUV_REPROCESSING)) return true + return false +} + +fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, + qualityPrioritization: QualityPrioritization, + flashMode: FlashMode, + enableRedEyeReduction: Boolean, + enableAutoStabilization: Boolean): CaptureRequest { + val cameraCharacteristics = cameraManager.getCameraCharacteristics(this.id) + val capabilities = cameraCharacteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES)!! + + val captureRequest = when (qualityPrioritization) { + // If speed, create application-specific Zero-Shutter-Lag template + QualityPrioritization.SPEED -> this.createCaptureRequest(CameraDevice.TEMPLATE_ZERO_SHUTTER_LAG) + // Otherwise create standard still image capture template + else -> this.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE) + } + if (qualityPrioritization == QualityPrioritization.SPEED) { + // Some devices also support hardware Zero-Shutter-Lag, try enabling that + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && supportsDeviceZsl(capabilities)) { + captureRequest[CaptureRequest.CONTROL_ENABLE_ZSL] = true + } + } + + // TODO: Maybe we can even expose that prop directly? + val jpegQuality = when (qualityPrioritization) { + QualityPrioritization.SPEED -> 85 + QualityPrioritization.BALANCED -> 92 + QualityPrioritization.QUALITY -> 100 + }.toByte() + captureRequest[CaptureRequest.JPEG_QUALITY] = jpegQuality + + // TODO: CaptureRequest.JPEG_ORIENTATION maybe? + + when (flashMode) { + // Set the Flash Mode + FlashMode.OFF -> { + captureRequest[CaptureRequest.FLASH_MODE] = CaptureRequest.FLASH_MODE_OFF + captureRequest[CaptureRequest.CONTROL_AE_MODE] = CaptureRequest.CONTROL_AE_MODE_ON + } + FlashMode.ON -> { + captureRequest[CaptureRequest.FLASH_MODE] = CaptureRequest.FLASH_MODE_SINGLE + captureRequest[CaptureRequest.CONTROL_AE_MODE] = CaptureRequest.CONTROL_AE_MODE_ON_ALWAYS_FLASH + } + FlashMode.AUTO -> { + if (enableRedEyeReduction) { + captureRequest[CaptureRequest.CONTROL_AE_MODE] = CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH_REDEYE + } else { + captureRequest[CaptureRequest.CONTROL_AE_MODE] = CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH + } + } + } + + if (enableAutoStabilization) { + // Enable optical or digital image stabilization + val digitalStabilization = cameraCharacteristics.get(CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES) + val hasDigitalStabilization = digitalStabilization?.contains(CameraCharacteristics.CONTROL_VIDEO_STABILIZATION_MODE_ON) ?: false + + val opticalStabilization = cameraCharacteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_OPTICAL_STABILIZATION) + val hasOpticalStabilization = opticalStabilization?.contains(CameraCharacteristics.LENS_OPTICAL_STABILIZATION_MODE_ON) ?: false + if (hasOpticalStabilization) { + captureRequest[CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE] = CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_OFF + captureRequest[CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE] = CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE_ON + } else if (hasDigitalStabilization) { + captureRequest[CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE] = CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON + } else { + // no stabilization is supported. ignore it + } + } + + return captureRequest.build() +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/ExifUtils.kt b/android/src/main/java/com/mrousavy/camera/utils/ExifUtils.kt new file mode 100644 index 0000000000..04984d4261 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/ExifUtils.kt @@ -0,0 +1,20 @@ +package com.mrousavy.camera.utils + +import androidx.exifinterface.media.ExifInterface + +class ExifUtils { + companion object { + fun computeExifOrientation(rotationDegrees: Int, mirrored: Boolean) = when { + rotationDegrees == 0 && !mirrored -> ExifInterface.ORIENTATION_NORMAL + rotationDegrees == 0 && mirrored -> ExifInterface.ORIENTATION_FLIP_HORIZONTAL + rotationDegrees == 180 && !mirrored -> ExifInterface.ORIENTATION_ROTATE_180 + rotationDegrees == 180 && mirrored -> ExifInterface.ORIENTATION_FLIP_VERTICAL + rotationDegrees == 270 && mirrored -> ExifInterface.ORIENTATION_TRANSVERSE + rotationDegrees == 90 && !mirrored -> ExifInterface.ORIENTATION_ROTATE_90 + rotationDegrees == 90 && mirrored -> ExifInterface.ORIENTATION_TRANSPOSE + rotationDegrees == 270 && mirrored -> ExifInterface.ORIENTATION_ROTATE_270 + rotationDegrees == 270 && !mirrored -> ExifInterface.ORIENTATION_TRANSVERSE + else -> ExifInterface.ORIENTATION_UNDEFINED + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt b/android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt new file mode 100644 index 0000000000..84af571e8b --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt @@ -0,0 +1,26 @@ +package com.mrousavy.camera.utils; + +import android.media.Image +import kotlinx.coroutines.CompletableDeferred + +class PhotoOutputSynchronizer { + private val photoOutputQueue = HashMap>() + + operator fun get(key: Long): CompletableDeferred { + if (!photoOutputQueue.containsKey(key)) { + photoOutputQueue[key] = CompletableDeferred() + } + return photoOutputQueue[key]!! + } + + fun set(key: Long, image: Image) { + this[key].complete(image) + } + + fun clear() { + photoOutputQueue.forEach { + it.value.cancel() + } + photoOutputQueue.clear() + } +} diff --git a/example/android/app/build.gradle b/example/android/app/build.gradle index 7029ff85e2..206bfec036 100644 --- a/example/android/app/build.gradle +++ b/example/android/app/build.gradle @@ -169,7 +169,6 @@ dependencies { } implementation project(':react-native-vision-camera') - implementation "androidx.camera:camera-core:1.1.0-alpha08" } apply from: file("../../node_modules/@react-native-community/cli-platform-android/native_modules.gradle"); applyNativeModulesAppBuildGradle(project) diff --git a/src/CameraDevice.ts b/src/CameraDevice.ts index 1ecbbe6b45..a93d06f875 100644 --- a/src/CameraDevice.ts +++ b/src/CameraDevice.ts @@ -52,12 +52,12 @@ export const parsePhysicalDeviceTypes = ( export type AutoFocusSystem = 'contrast-detection' | 'phase-detection' | 'none'; /** - * Indicates a format's supported video stabilization mode + * Indicates a format's supported video stabilization mode. Enabling video stabilization may introduce additional latency into the video capture pipeline. * - * * `"off"`: Indicates that video should not be stabilized - * * `"standard"`: Indicates that video should be stabilized using the standard video stabilization algorithm introduced with iOS 5.0. Standard video stabilization has a reduced field of view. Enabling video stabilization may introduce additional latency into the video capture pipeline - * * `"cinematic"`: Indicates that video should be stabilized using the cinematic stabilization algorithm for more dramatic results. Cinematic video stabilization has a reduced field of view compared to standard video stabilization. Enabling cinematic video stabilization introduces much more latency into the video capture pipeline than standard video stabilization and consumes significantly more system memory. Use narrow or identical min and max frame durations in conjunction with this mode - * * `"cinematic-extended"`: Indicates that the video should be stabilized using the extended cinematic stabilization algorithm. Enabling extended cinematic stabilization introduces longer latency into the video capture pipeline compared to the AVCaptureVideoStabilizationModeCinematic and consumes more memory, but yields improved stability. It is recommended to use identical or similar min and max frame durations in conjunction with this mode (iOS 13.0+) + * * `"off"`: No video stabilization. Indicates that video should not be stabilized + * * `"standard"`: Standard software-based video stabilization. Standard video stabilization reduces the field of view by about 10%. + * * `"cinematic"`: Advanced software-based video stabilization. This applies more aggressive cropping or transformations than standard. + * * `"cinematic-extended"`: Extended software- and hardware-based stabilization that aggressively crops and transforms the video to apply a smooth cinematic stabilization. * * `"auto"`: Indicates that the most appropriate video stabilization mode for the device and format should be chosen automatically */ export type VideoStabilizationMode = 'off' | 'standard' | 'cinematic' | 'cinematic-extended' | 'auto'; diff --git a/src/PhotoFile.ts b/src/PhotoFile.ts index bc539c35e9..21ad8391a1 100644 --- a/src/PhotoFile.ts +++ b/src/PhotoFile.ts @@ -9,7 +9,6 @@ export interface TakePhotoOptions { * * `"balanced"` Indicates that photo quality and speed of delivery are balanced in priority * * `"speed"` Indicates that speed of photo delivery is most important, even at the expense of quality * - * @platform iOS 13.0+ * @default "balanced" */ qualityPrioritization?: 'quality' | 'balanced' | 'speed'; @@ -32,13 +31,17 @@ export interface TakePhotoOptions { */ enableAutoStabilization?: boolean; /** - * Specifies whether the photo output should use content aware distortion correction on this photo request (at its discretion). + * Specifies whether the photo output should use content aware distortion correction on this photo request. + * For example, the algorithm may not apply correction to faces in the center of a photo, but may apply it to faces near the photo’s edges. * + * @platform iOS * @default false */ enableAutoDistortionCorrection?: boolean; /** * Specifies the photo codec to use for this capture. The provided photo codec has to be supported by the session. + * + * @platform iOS */ photoCodec?: CameraPhotoCodec; /** @@ -69,7 +72,7 @@ export interface PhotoFile extends TemporaryFile { * @see [AVCapturePhoto.metadata](https://developer.apple.com/documentation/avfoundation/avcapturephoto/2873982-metadata) * @see [AndroidX ExifInterface](https://developer.android.com/reference/androidx/exifinterface/media/ExifInterface) */ - metadata: { + metadata?: { Orientation: number; /** * @platform iOS From 645bc1abad812b07d21b9fa53205ad3cda76f6dd Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 20:54:06 +0200 Subject: [PATCH 035/180] Add ExifInterface library --- android/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/android/build.gradle b/android/build.gradle index 5ebb757189..164848297a 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -146,6 +146,7 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.5.2" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2" + implementation "androidx.exifinterface:exifinterface:1.3.6" implementation project(":react-native-worklets") implementation project(":shopify_react-native-skia") From a83c596f5f51130dc5a8ac0ff3076f2339d21517 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:07:28 +0200 Subject: [PATCH 036/180] Run findViewById on UI Thread --- .../com/mrousavy/camera/CameraViewModule.kt | 50 ++++++++++++------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index 3eebed6b32..4dc945a2e1 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -21,6 +21,9 @@ import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlinx.coroutines.guava.await import java.util.concurrent.Executors +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine @ReactModule(name = CameraViewModule.TAG) @Suppress("unused") @@ -43,11 +46,16 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ return TAG } - private fun findCameraView(viewId: Int): CameraView { - Log.d(TAG, "Finding view $viewId...") - val view = if (reactApplicationContext != null) UIManagerHelper.getUIManager(reactApplicationContext, viewId)?.resolveView(viewId) as CameraView? else null - Log.d(TAG, if (reactApplicationContext != null) "Found view $viewId!" else "Couldn't find view $viewId!") - return view ?: throw ViewNotFoundError(viewId) + private suspend fun findCameraView(viewId: Int): CameraView { + return suspendCoroutine { continuation -> + UiThreadUtil.runOnUiThread { + Log.d(TAG, "Finding view $viewId...") + val view = if (reactApplicationContext != null) UIManagerHelper.getUIManager(reactApplicationContext, viewId)?.resolveView(viewId) as CameraView? else null + Log.d(TAG, if (reactApplicationContext != null) "Found view $viewId!" else "Couldn't find view $viewId!") + if (view != null) continuation.resume(view) + else continuation.resumeWithException(ViewNotFoundError(viewId)) + } + } } @ReactMethod(isBlockingSynchronousMethod = true) @@ -65,8 +73,8 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod fun takePhoto(viewTag: Int, options: ReadableMap, promise: Promise) { coroutineScope.launch { + val view = findCameraView(viewTag) withPromise(promise) { - val view = findCameraView(viewTag) view.takePhoto(options) } } @@ -76,8 +84,8 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod fun takeSnapshot(viewTag: Int, options: ReadableMap, promise: Promise) { coroutineScope.launch { + val view = findCameraView(viewTag) withPromise(promise) { - val view = findCameraView(viewTag) view.takeSnapshot(options) } } @@ -102,36 +110,42 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod fun pauseRecording(viewTag: Int, promise: Promise) { - withPromise(promise) { - val view = findCameraView(viewTag) - view.pauseRecording() - return@withPromise null + coroutineScope.launch { + withPromise(promise) { + val view = findCameraView(viewTag) + view.pauseRecording() + return@withPromise null + } } } @ReactMethod fun resumeRecording(viewTag: Int, promise: Promise) { - withPromise(promise) { + coroutineScope.launch { val view = findCameraView(viewTag) - view.resumeRecording() - return@withPromise null + withPromise(promise) { + view.resumeRecording() + return@withPromise null + } } } @ReactMethod fun stopRecording(viewTag: Int, promise: Promise) { - withPromise(promise) { + coroutineScope.launch { val view = findCameraView(viewTag) - view.stopRecording() - return@withPromise null + withPromise(promise) { + view.stopRecording() + return@withPromise null + } } } @ReactMethod fun focus(viewTag: Int, point: ReadableMap, promise: Promise) { coroutineScope.launch { + val view = findCameraView(viewTag) withPromise(promise) { - val view = findCameraView(viewTag) view.focus(point) return@withPromise null } From dbe34cef40ce5daf52edbee0c88e3fe7605b7ec1 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:07:40 +0200 Subject: [PATCH 037/180] Add Photo Output Surface to takePhoto --- android/src/main/java/com/mrousavy/camera/CameraSession.kt | 2 ++ .../camera/utils/CameraDevice+createPhotoCaptureRequest.kt | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 0b7839a306..266c70b8a7 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -161,7 +161,9 @@ class CameraSession(private val cameraManager: CameraManager, enableAutoStabilization: Boolean): CapturedPhoto { val captureSession = captureSession ?: throw CameraNotReadyError() + val photoOutput = outputs.find { it.outputType == OutputType.PHOTO } ?: throw PhotoNotEnabledError() val captureRequest = captureSession.device.createPhotoCaptureRequest(cameraManager, + photoOutput.surface, qualityPrioritization, flashMode, enableRedEyeReduction, diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt index c9c876d553..edf5ab5d75 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt @@ -5,6 +5,7 @@ import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.os.Build +import android.view.Surface enum class FlashMode { OFF, ON, AUTO } @@ -18,6 +19,7 @@ private fun supportsDeviceZsl(capabilities: IntArray): Boolean { } fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, + surface: Surface, qualityPrioritization: QualityPrioritization, flashMode: FlashMode, enableRedEyeReduction: Boolean, @@ -84,5 +86,7 @@ fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, } } + captureRequest.addTarget(surface) + return captureRequest.build() } From 5c93aa8201e1d2c470afca79e3ee6a2f85dc4012 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:07:45 +0200 Subject: [PATCH 038/180] Fix Video Stabilization Modes --- .../com/mrousavy/camera/utils/CameraDeviceDetails.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 657e423744..791b10deb9 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -12,10 +12,11 @@ import android.util.Size import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap +import com.mrousavy.camera.parsers.parseDigitalVideoStabilizationMode import com.mrousavy.camera.parsers.parseHardwareLevel import com.mrousavy.camera.parsers.parseImageFormat import com.mrousavy.camera.parsers.parseLensFacing -import com.mrousavy.camera.parsers.parseVideoStabilizationMode +import com.mrousavy.camera.parsers.parseOpticalVideoStabilizationMode import kotlin.math.PI import kotlin.math.atan @@ -77,9 +78,11 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private fun createStabilizationModes(): ReadableArray { val array = Arguments.createArray() - val videoStabilizationModes = digitalStabilizationModes.plus(opticalStabilizationModes) - videoStabilizationModes.forEach { videoStabilizationMode -> - array.pushString(parseVideoStabilizationMode(videoStabilizationMode)) + digitalStabilizationModes.forEach { videoStabilizationMode -> + array.pushString(parseDigitalVideoStabilizationMode(videoStabilizationMode)) + } + opticalStabilizationModes.forEach { videoStabilizationMode -> + array.pushString(parseOpticalVideoStabilizationMode(videoStabilizationMode)) } return array } From 16c3fd107ecbfe73ce07c2176538c017f01f7973 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:08:57 +0200 Subject: [PATCH 039/180] Optimize Imports --- .../src/main/java/com/mrousavy/camera/CameraQueues.kt | 4 ---- .../java/com/mrousavy/camera/CameraView+RecordVideo.kt | 1 - .../java/com/mrousavy/camera/CameraView+TakePhoto.kt | 5 ----- .../src/main/java/com/mrousavy/camera/CameraView.kt | 4 ---- .../main/java/com/mrousavy/camera/CameraViewModule.kt | 4 ---- android/src/main/java/com/mrousavy/camera/Errors.kt | 2 -- .../main/java/com/mrousavy/camera/NativePreviewView.kt | 2 -- .../camera/frameprocessor/VisionCameraProxy.kt | 1 - .../camera/parsers/VideoStabilizationMode+String.kt | 7 +++++-- .../camera/utils/CameraDevice+createCaptureSession.kt | 10 ---------- 10 files changed, 5 insertions(+), 35 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt index 9360d4a735..a5b268e2d6 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt @@ -4,12 +4,8 @@ import android.os.Handler import android.os.HandlerThread import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.android.asCoroutineDispatcher -import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.asExecutor import java.util.concurrent.Executor -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import kotlin.coroutines.CoroutineContext class CameraQueues { companion object { diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt index d4310ff65a..5997377098 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt @@ -3,7 +3,6 @@ package com.mrousavy.camera import android.Manifest import android.annotation.SuppressLint import android.content.pm.PackageManager -import android.media.Image import android.util.Log import androidx.core.content.ContextCompat import com.facebook.react.bridge.* diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index 5ac65a32c5..8b9ced2aca 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -7,15 +7,10 @@ import android.hardware.camera2.* import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap -import com.mrousavy.camera.parsers.getVideoStabilizationMode import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import java.io.File import java.io.FileOutputStream -import java.nio.file.Files -import java.text.SimpleDateFormat -import java.util.Date -import java.util.Locale @SuppressLint("UnsafeOptInUsageError") suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = coroutineScope { diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 71d9c27bae..5c9c093007 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -9,7 +9,6 @@ import android.hardware.camera2.CameraManager import android.util.Log import android.util.Size import android.view.Surface -import android.view.SurfaceHolder import android.view.SurfaceView import android.view.View import android.widget.FrameLayout @@ -17,12 +16,9 @@ import androidx.core.content.ContextCompat import com.facebook.react.bridge.ReadableMap import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor -import com.mrousavy.camera.hooks.UseCameraDevice -import com.mrousavy.camera.hooks.UseSurfaceViewSurface import com.mrousavy.camera.utils.containsAny import com.mrousavy.camera.utils.displayRotation import com.mrousavy.camera.utils.installHierarchyFitter -import kotlinx.coroutines.launch import kotlin.math.max import kotlin.math.min diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index 4dc945a2e1..59030c9d58 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -12,15 +12,11 @@ import com.facebook.react.module.annotations.ReactModule import com.facebook.react.modules.core.PermissionAwareActivity import com.facebook.react.modules.core.PermissionListener import com.facebook.react.uimanager.UIManagerHelper -import com.facebook.react.bridge.ReactApplicationContext import com.mrousavy.camera.frameprocessor.VisionCameraInstaller -import java.util.concurrent.ExecutorService import com.mrousavy.camera.frameprocessor.VisionCameraProxy import com.mrousavy.camera.parsers.* import com.mrousavy.camera.utils.* import kotlinx.coroutines.* -import kotlinx.coroutines.guava.await -import java.util.concurrent.Executors import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 0a9d62f513..caffc68dd0 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -1,7 +1,5 @@ package com.mrousavy.camera -import android.graphics.ImageFormat - abstract class CameraError( /** * The domain of the error. Error domains are used to group errors. diff --git a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt index 6c0650a09c..40d0cae321 100644 --- a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt +++ b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt @@ -3,7 +3,6 @@ package com.mrousavy.camera import android.annotation.SuppressLint import android.content.Context import android.content.res.Resources -import android.graphics.ImageFormat import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.util.Log @@ -13,7 +12,6 @@ import android.view.SurfaceHolder import android.view.SurfaceView import com.mrousavy.camera.utils.bigger import com.mrousavy.camera.utils.smaller -import kotlin.math.max import kotlin.math.roundToInt /** diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index a44c1fc209..30b42e8c8a 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -13,7 +13,6 @@ import com.facebook.react.uimanager.UIManagerHelper import com.mrousavy.camera.CameraView import com.mrousavy.camera.ViewNotFoundError import java.lang.ref.WeakReference -import java.util.concurrent.ExecutorService @Suppress("KotlinJniMissingFunction") // we use fbjni. diff --git a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt index 7dbaaea665..171633d246 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt @@ -1,7 +1,10 @@ package com.mrousavy.camera.parsers -import android.hardware.camera2.CameraMetadata.* -import android.os.Build +import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_OFF +import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_ON +import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION +import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_OFF +import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_ON data class VideoStabilizationMode(val digitalMode: Int, val opticalMode: Int) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 52ef822e64..4181956760 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -1,26 +1,16 @@ package com.mrousavy.camera.utils -import android.content.res.Resources import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration -import android.media.CamcorderProfile import android.os.Build import android.util.Log -import android.util.Size import android.view.Surface import com.mrousavy.camera.CameraQueues -import com.mrousavy.camera.CameraView import com.mrousavy.camera.parsers.parseHardwareLevel -import kotlinx.coroutines.android.asCoroutineDispatcher -import kotlinx.coroutines.asExecutor -import java.util.concurrent.Executor -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlin.coroutines.suspendCoroutine enum class SessionType { REGULAR, From 496e36d6b299b6815e25b4aa04d2111f2433e31b Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:18:15 +0200 Subject: [PATCH 040/180] More logs --- .../src/main/java/com/mrousavy/camera/CameraSession.kt | 4 ++++ .../java/com/mrousavy/camera/CameraView+TakePhoto.kt | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 266c70b8a7..363912f0bf 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -168,7 +168,9 @@ class CameraSession(private val cameraManager: CameraManager, flashMode, enableRedEyeReduction, enableAutoStabilization) + Log.i(TAG, "Capturing Photo...") val result = captureSession.capture(captureRequest) + Log.i(TAG, "Photo captured! Awaiting image data..") val timestamp = result[CaptureResult.SENSOR_TIMESTAMP]!! try { val image = photoOutputSynchronizer[timestamp].await() @@ -248,6 +250,7 @@ class CameraSession(private val cameraManager: CameraManager, cameraDevice = null captureSession?.close() captureSession = null + photoOutputSynchronizer.clear() } @@ -359,6 +362,7 @@ class CameraSession(private val cameraManager: CameraManager, } catch (_: Throwable) { } captureSession = null + photoOutputSynchronizer.clear() } }, CameraQueues.cameraQueue diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index 8b9ced2aca..26edfd78c0 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -4,6 +4,7 @@ import android.annotation.SuppressLint import android.content.Context import android.graphics.ImageFormat import android.hardware.camera2.* +import android.util.Log import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap @@ -12,10 +13,12 @@ import kotlinx.coroutines.* import java.io.File import java.io.FileOutputStream +private const val TAG = "CameraView.takePhoto" + @SuppressLint("UnsafeOptInUsageError") suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = coroutineScope { - // TODO: takePhoto() val options = optionsMap.toHashMap() + Log.i(TAG, "Taking photo... Options: $options") val qualityPrioritization = options["qualityPrioritization"] as? String val flash = options["flash"] as? String @@ -41,10 +44,14 @@ suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = corouti enableAutoRedEyeReduction, enableAutoStabilization) + Log.i(TAG, "Successfully captured ${photo.image.width} x ${photo.image.height} photo!") + val cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraId!!) val path = savePhotoToFile(context, cameraCharacteristics, photo) + Log.i(TAG, "Successfully saved photo to file! $path") + val map = Arguments.createMap() map.putString("path", path) map.putInt("width", photo.image.width) From cb80f73cb35e016fd960e58709cf2c953b662ae9 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:35:12 +0200 Subject: [PATCH 041/180] Update CameraSession.kt --- android/src/main/java/com/mrousavy/camera/CameraSession.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 363912f0bf..6fbe0bd085 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -168,15 +168,16 @@ class CameraSession(private val cameraManager: CameraManager, flashMode, enableRedEyeReduction, enableAutoStabilization) - Log.i(TAG, "Capturing Photo...") + Log.i(TAG, "Photo capture 0/2 - starting capture...") val result = captureSession.capture(captureRequest) - Log.i(TAG, "Photo captured! Awaiting image data..") val timestamp = result[CaptureResult.SENSOR_TIMESTAMP]!! + Log.i(TAG, "Photo capture 1/2 complete - received metadata with timestamp $timestamp") try { val image = photoOutputSynchronizer[timestamp].await() // TODO: Correctly get rotationDegrees and isMirrored val rotation = ExifUtils.computeExifOrientation(0, false) + Log.i(TAG, "Photo capture 2/2 complete - received ${image.width} x ${image.height} image.") return CapturedPhoto(image, result, rotation, image.format) } catch (e: CancellationException) { throw CaptureAbortedError(false) From fb6cb4457bf10403c0044b1d4c419db882d77cf2 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:37:54 +0200 Subject: [PATCH 042/180] Close Image --- .../java/com/mrousavy/camera/CameraSession.kt | 6 ++++- .../mrousavy/camera/CameraView+TakePhoto.kt | 24 ++++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 6fbe0bd085..1e4b3abb8b 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -67,7 +67,11 @@ class CameraSession(private val cameraManager: CameraManager, data class CapturedPhoto(val image: Image, val metadata: TotalCaptureResult, val orientation: Int, - val format: Int) + val format: Int): Closeable { + override fun close() { + image.close() + } + } // setInput(..) private var cameraId: String? = null diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index 26edfd78c0..b59d217650 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -44,23 +44,25 @@ suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = corouti enableAutoRedEyeReduction, enableAutoStabilization) - Log.i(TAG, "Successfully captured ${photo.image.width} x ${photo.image.height} photo!") + photo.use { + Log.i(TAG, "Successfully captured ${photo.image.width} x ${photo.image.height} photo!") - val cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraId!!) + val cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraId!!) - val path = savePhotoToFile(context, cameraCharacteristics, photo) + val path = savePhotoToFile(context, cameraCharacteristics, photo) - Log.i(TAG, "Successfully saved photo to file! $path") + Log.i(TAG, "Successfully saved photo to file! $path") - val map = Arguments.createMap() - map.putString("path", path) - map.putInt("width", photo.image.width) - map.putInt("height", photo.image.height) - map.putBoolean("isRawPhoto", photo.format == ImageFormat.RAW_SENSOR) + val map = Arguments.createMap() + map.putString("path", path) + map.putInt("width", photo.image.width) + map.putInt("height", photo.image.height) + map.putBoolean("isRawPhoto", photo.format == ImageFormat.RAW_SENSOR) - // TODO: Add metadata prop to resulting photo + // TODO: Add metadata prop to resulting photo - return@coroutineScope map + return@coroutineScope map + } } private suspend fun savePhotoToFile(context: Context, From cdeaf01375327edcdc5d8fc7c3fa2bffec3afc8f Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:51:54 +0200 Subject: [PATCH 043/180] Use separate Executor in CameraQueue --- .../java/com/mrousavy/camera/CameraQueues.kt | 9 +--- .../java/com/mrousavy/camera/CameraSession.kt | 6 ++- .../camera/utils/CameraManager+openCamera.kt | 53 ------------------- 3 files changed, 7 insertions(+), 61 deletions(-) delete mode 100644 android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt index a5b268e2d6..e05de2008e 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt @@ -2,10 +2,8 @@ package com.mrousavy.camera import android.os.Handler import android.os.HandlerThread -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.android.asCoroutineDispatcher -import kotlinx.coroutines.asExecutor import java.util.concurrent.Executor +import java.util.concurrent.Executors class CameraQueues { companion object { @@ -15,7 +13,6 @@ class CameraQueues { class CameraQueue(name: String) { val handler: Handler - val coroutineScope: CoroutineScope private val thread: HandlerThread val executor: Executor @@ -23,9 +20,7 @@ class CameraQueues { thread = HandlerThread(name) thread.start() handler = Handler(thread.looper) - val coroutineDispatcher = handler.asCoroutineDispatcher() - coroutineScope = CoroutineScope(coroutineDispatcher) - executor = coroutineDispatcher.asExecutor() + executor = Executors.newSingleThreadExecutor() } protected fun finalize() { diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 1e4b3abb8b..fb272e795b 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -97,7 +97,11 @@ class CameraSession(private val cameraManager: CameraManager, private val photoOutputSynchronizer = PhotoOutputSynchronizer() init { - cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + cameraManager.registerAvailabilityCallback(CameraQueues.cameraQueue.executor, this) + } else { + cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) + } } override fun close() { diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt deleted file mode 100644 index eaaf7e8e5b..0000000000 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraManager+openCamera.kt +++ /dev/null @@ -1,53 +0,0 @@ -package com.mrousavy.camera.utils - -import android.annotation.SuppressLint -import android.hardware.camera2.CameraDevice -import android.hardware.camera2.CameraManager -import android.util.Log -import com.mrousavy.camera.CameraCannotBeOpenedError -import com.mrousavy.camera.CameraDisconnectedError -import com.mrousavy.camera.CameraQueues -import com.mrousavy.camera.CameraView -import com.mrousavy.camera.parsers.parseCameraError -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlin.coroutines.suspendCoroutine - -@SuppressLint("MissingPermission") -suspend fun CameraManager.openCamera(cameraId: String, onClosed: () -> Unit): CameraDevice { - return suspendCoroutine { continuation -> - var didRun = false - Log.i(CameraView.TAG, "Opening Camera $cameraId...") - - - this.openCamera(cameraId, object: CameraDevice.StateCallback() { - override fun onOpened(device: CameraDevice) { - Log.i(CameraView.TAG, "Successfully opened Camera Device $cameraId!") - if (!didRun) { - continuation.resume(device) - didRun = true - } - } - - override fun onDisconnected(camera: CameraDevice) { - Log.w(CameraView.TAG, "Camera Device $cameraId has been disconnected! Closing Camera..") - if (!didRun) { - continuation.resumeWithException(CameraDisconnectedError(cameraId)) - didRun = true - } else { - onClosed() - } - } - - override fun onError(camera: CameraDevice, errorCode: Int) { - Log.e(CameraView.TAG, "Failed to open Camera Device $cameraId! Closing Camera.. Error: $errorCode (${parseCameraError(errorCode)})") - if (!didRun) { - continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, parseCameraError(errorCode))) - didRun = true - } else { - onClosed() - } - } - }, CameraQueues.cameraQueue.handler) - } -} From aa694c15ac8e52da30dc5567f65e2a6a10f7a866 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 21:56:22 +0200 Subject: [PATCH 044/180] Delete hooks --- .../com/mrousavy/camera/hooks/HookListener.kt | 15 ---- .../mrousavy/camera/hooks/UseCameraDevice.kt | 74 ------------------- .../camera/hooks/UseSurfaceViewSurface.kt | 38 ---------- 3 files changed, 127 deletions(-) delete mode 100644 android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt diff --git a/android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt b/android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt deleted file mode 100644 index c7ac23be71..0000000000 --- a/android/src/main/java/com/mrousavy/camera/hooks/HookListener.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.mrousavy.camera.hooks - - -abstract class DataProvider(private val onChange: (value: T?) -> Unit) { - private var value: T? = null - fun update(value: T?) { - if (this.value != value) { - this.value = value - onChange(value) - } - } - - val currentValue: T? - get() = value -} diff --git a/android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt b/android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt deleted file mode 100644 index fd51b12d15..0000000000 --- a/android/src/main/java/com/mrousavy/camera/hooks/UseCameraDevice.kt +++ /dev/null @@ -1,74 +0,0 @@ -package com.mrousavy.camera.hooks - -import android.annotation.SuppressLint -import android.hardware.camera2.CameraDevice -import android.hardware.camera2.CameraManager -import android.util.Log -import com.mrousavy.camera.CameraQueues -import com.mrousavy.camera.CameraView -import com.mrousavy.camera.parsers.parseCameraError -import java.io.Closeable - -class UseCameraDevice(private val cameraManager: CameraManager, - val cameraId: String, - onChange: (device: CameraDevice?) -> Unit): Closeable, DataProvider(onChange) { - - private var isOpening = false - private val availabilityCallback = object: CameraManager.AvailabilityCallback() { - override fun onCameraAvailable(id: String) { - super.onCameraAvailable(id) - if (id == cameraId) { - // Our camera is available, try to open it - openCamera() - } - } - - override fun onCameraUnavailable(id: String) { - super.onCameraUnavailable(id) - if (id == cameraId) { - // Our camera is no longer available - update(null) - } - } - } - private val openCameraCallback = object: CameraDevice.StateCallback() { - override fun onOpened(camera: CameraDevice) { - isOpening = false - Log.i(CameraView.TAG, "Successfully opened Camera Device $cameraId!") - update(camera) - } - - override fun onDisconnected(camera: CameraDevice) { - isOpening = false - Log.w(CameraView.TAG, "Camera Device $cameraId has been disconnected! Closing Camera..") - camera.close() - update(null) - } - - override fun onError(camera: CameraDevice, errorCode: Int) { - isOpening = false - Log.e(CameraView.TAG, "Failed to open Camera Device $cameraId! Closing Camera.. " + - "Error: $errorCode (${parseCameraError(errorCode)})") - camera.close() - update(null) - } - } - - init { - cameraManager.registerAvailabilityCallback(availabilityCallback, CameraQueues.cameraQueue.handler) - } - - @SuppressLint("MissingPermission") - fun openCamera() { - if (isOpening || currentValue?.id == cameraId) { - // camera is already opened, no need to re-open. - return - } - isOpening = true - cameraManager.openCamera(cameraId, openCameraCallback, CameraQueues.cameraQueue.handler) - } - - override fun close() { - cameraManager.unregisterAvailabilityCallback(availabilityCallback) - } -} diff --git a/android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt b/android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt deleted file mode 100644 index 021f187f9c..0000000000 --- a/android/src/main/java/com/mrousavy/camera/hooks/UseSurfaceViewSurface.kt +++ /dev/null @@ -1,38 +0,0 @@ -package com.mrousavy.camera.hooks - -import android.util.Log -import android.view.Surface -import android.view.SurfaceHolder -import android.view.SurfaceView -import java.io.Closeable - -class UseSurfaceViewSurface(private val surfaceView: SurfaceView, - onChange: (surface: Surface?) -> Unit): Closeable, DataProvider(onChange) { - companion object { - private const val TAG = "UseSurfaceViewSurface" - } - - private val surfaceCallback = object: SurfaceHolder.Callback { - override fun surfaceCreated(holder: SurfaceHolder) { - Log.d(TAG, "Surface Created: ${holder.surface}") - update(holder.surface) - } - - override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { - Log.d(TAG, "Surface resized: ${holder.surface} $format $width x $height") - } - - override fun surfaceDestroyed(holder: SurfaceHolder) { - Log.d(TAG, "Surface Destroyed: ${holder.surface}") - update(null) - } - } - - init { - surfaceView.holder.addCallback(surfaceCallback) - } - - override fun close() { - surfaceView.holder.removeCallback(surfaceCallback) - } -} From 4c5c724b7b9b60be2f5026a03db50ff34c965d81 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 22:00:27 +0200 Subject: [PATCH 045/180] Use same Thread again --- android/src/main/java/com/mrousavy/camera/CameraQueues.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt index e05de2008e..ea538b7603 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt @@ -2,8 +2,9 @@ package com.mrousavy.camera import android.os.Handler import android.os.HandlerThread +import kotlinx.coroutines.android.asCoroutineDispatcher +import kotlinx.coroutines.asExecutor import java.util.concurrent.Executor -import java.util.concurrent.Executors class CameraQueues { companion object { @@ -20,7 +21,8 @@ class CameraQueues { thread = HandlerThread(name) thread.start() handler = Handler(thread.looper) - executor = Executors.newSingleThreadExecutor() + val coroutineDispatcher = handler.asCoroutineDispatcher(name) + executor = coroutineDispatcher.asExecutor() } protected fun finalize() { From fe8ef662495204f91118ac7a96d696a7d99c4ea4 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 22:00:40 +0200 Subject: [PATCH 046/180] If opened, call error --- .../java/com/mrousavy/camera/CameraSession.kt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index fb272e795b..19cfd3d709 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -223,10 +223,12 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Opening Camera $cameraId...") + var didOpen = false cameraManager.openCamera(cameraId, object: CameraDevice.StateCallback() { // When Camera is successfully opened (called once) override fun onOpened(camera: CameraDevice) { Log.i(TAG, "Camera $cameraId: opened!") + didOpen = true onCameraInitialized(camera) } @@ -234,15 +236,23 @@ class CameraSession(private val cameraManager: CameraManager, override fun onDisconnected(camera: CameraDevice) { Log.i(TAG, "Camera $cameraId: disconnected!") + if (didOpen) { + onError(CameraDisconnectedError(camera.id)) + } + onCameraDisconnected() camera.close() } // When Camera has been encountered an Error (either called on init, or later) override fun onError(camera: CameraDevice, errorCode: Int) { - val errorString = parseCameraError(errorCode) - onError(CameraCannotBeOpenedError(cameraId, errorString)) - Log.e(TAG, "Camera $cameraId: error! ($errorCode: $errorString)") + Log.i(TAG, "onError(${camera.id}) $errorCode") + + if (didOpen) { + val errorString = parseCameraError(errorCode) + onError(CameraCannotBeOpenedError(cameraId, errorString)) + Log.e(TAG, "Camera $cameraId: error! ($errorCode: $errorString)") + } onCameraDisconnected() camera.close() From c3a391cc5a162759ce3e513fa15729b234269da4 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 22:02:12 +0200 Subject: [PATCH 047/180] Update CameraSession.kt --- .../src/main/java/com/mrousavy/camera/CameraSession.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 19cfd3d709..0df1aded2e 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -238,6 +238,8 @@ class CameraSession(private val cameraManager: CameraManager, if (didOpen) { onError(CameraDisconnectedError(camera.id)) + } else { + onError(CameraCannotBeOpenedError(camera.id, "camera-disconnected")) } onCameraDisconnected() @@ -249,9 +251,9 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "onError(${camera.id}) $errorCode") if (didOpen) { - val errorString = parseCameraError(errorCode) - onError(CameraCannotBeOpenedError(cameraId, errorString)) - Log.e(TAG, "Camera $cameraId: error! ($errorCode: $errorString)") + onError(CameraDisconnectedError(camera.id)) + } else { + onError(CameraCannotBeOpenedError(camera.id, parseCameraError(errorCode))) } onCameraDisconnected() From 626438e4c3fd148a768f7ff771c84521a2e7c96d Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 22:16:30 +0200 Subject: [PATCH 048/180] Log HW level --- .../mrousavy/camera/utils/CameraDevice+createCaptureSession.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index 4181956760..c9569162f3 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -78,7 +78,7 @@ fun CameraDevice.createCaptureSession(cameraManager: CameraManager, queue: CameraQueues.CameraQueue) { val characteristics = cameraManager.getCameraCharacteristics(this.id) val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! - Log.i(TAG, "Creating Capture Session on ${parseHardwareLevel(hardwareLevel)} device...") + Log.i(TAG, "Creating Capture Session.. Hardware Level: ${parseHardwareLevel(hardwareLevel)}") if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { // API >= 24 From 224b44fe5ad04a481f92b3a9fa42ece6059e1067 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 3 Aug 2023 22:20:34 +0200 Subject: [PATCH 049/180] fix: Don't enable Stream Use Case if it's not 100% supported --- .../camera/utils/CameraDevice+createCaptureSession.kt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt index c9569162f3..442fb0272a 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt @@ -58,13 +58,6 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu } } } - // See https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture - // According to the Android Documentation, devices with LEVEL_3 or FULL support can do 4 use-cases. - // LIMITED or LEGACY devices can't do it. - val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! - if (hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3 || hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL) { - return true - } return false } From 8221fd256ac5d5f759e4c802168551bbc5bf49d1 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 09:25:31 +0200 Subject: [PATCH 050/180] Move some stuff --- .../java/com/mrousavy/camera/CameraSession.kt | 24 ++++++++----------- .../mrousavy/camera/CameraView+TakePhoto.kt | 2 ++ .../java/com/mrousavy/camera/CameraView.kt | 6 ++--- .../com/mrousavy/camera/NativePreviewView.kt | 4 ++-- .../CameraCaptureSession+capture.kt | 2 +- .../CameraDevice+createCaptureSession.kt | 2 +- .../CameraDevice+createPhotoCaptureRequest.kt | 2 +- .../Context+displayRotation.kt | 2 +- .../{utils => extensions}/List+containsAny.kt | 2 +- .../{utils => extensions}/Size+Extensions.kt | 2 +- .../ViewGroup+installHierarchyFitter.kt | 2 +- .../WritableArray+Nullables.kt | 2 +- .../WritableMap+Nullables.kt | 2 +- .../camera/utils/CameraDeviceDetails.kt | 1 + 14 files changed, 27 insertions(+), 28 deletions(-) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/CameraCaptureSession+capture.kt (97%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/CameraDevice+createCaptureSession.kt (99%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/CameraDevice+createPhotoCaptureRequest.kt (99%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/Context+displayRotation.kt (95%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/List+containsAny.kt (75%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/Size+Extensions.kt (96%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/ViewGroup+installHierarchyFitter.kt (95%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/WritableArray+Nullables.kt (91%) rename android/src/main/java/com/mrousavy/camera/{utils => extensions}/WritableMap+Nullables.kt (92%) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 0df1aded2e..93b8275176 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -17,19 +17,19 @@ import android.util.Log import android.util.Range import android.util.Size import android.view.Surface +import com.mrousavy.camera.extensions.FlashMode +import com.mrousavy.camera.extensions.OutputType +import com.mrousavy.camera.extensions.QualityPrioritization +import com.mrousavy.camera.extensions.SessionType +import com.mrousavy.camera.extensions.SurfaceOutput +import com.mrousavy.camera.extensions.capture +import com.mrousavy.camera.extensions.closestToOrMax +import com.mrousavy.camera.extensions.createCaptureSession +import com.mrousavy.camera.extensions.createPhotoCaptureRequest import com.mrousavy.camera.parsers.getVideoStabilizationMode import com.mrousavy.camera.parsers.parseCameraError import com.mrousavy.camera.utils.ExifUtils -import com.mrousavy.camera.utils.FlashMode -import com.mrousavy.camera.utils.OutputType import com.mrousavy.camera.utils.PhotoOutputSynchronizer -import com.mrousavy.camera.utils.QualityPrioritization -import com.mrousavy.camera.utils.SessionType -import com.mrousavy.camera.utils.SurfaceOutput -import com.mrousavy.camera.utils.capture -import com.mrousavy.camera.utils.closestToOrMax -import com.mrousavy.camera.utils.createCaptureSession -import com.mrousavy.camera.utils.createPhotoCaptureRequest import java.io.Closeable import java.util.concurrent.CancellationException @@ -97,11 +97,7 @@ class CameraSession(private val cameraManager: CameraManager, private val photoOutputSynchronizer = PhotoOutputSynchronizer() init { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - cameraManager.registerAvailabilityCallback(CameraQueues.cameraQueue.executor, this) - } else { - cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) - } + cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) } override fun close() { diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index b59d217650..291964949b 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -8,6 +8,8 @@ import android.util.Log import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap +import com.mrousavy.camera.extensions.FlashMode +import com.mrousavy.camera.extensions.QualityPrioritization import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import java.io.File diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 5c9c093007..37f908f9f7 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -16,9 +16,9 @@ import androidx.core.content.ContextCompat import com.facebook.react.bridge.ReadableMap import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor -import com.mrousavy.camera.utils.containsAny -import com.mrousavy.camera.utils.displayRotation -import com.mrousavy.camera.utils.installHierarchyFitter +import com.mrousavy.camera.extensions.containsAny +import com.mrousavy.camera.extensions.displayRotation +import com.mrousavy.camera.extensions.installHierarchyFitter import kotlin.math.max import kotlin.math.min diff --git a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt index 40d0cae321..39e0e10312 100644 --- a/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt +++ b/android/src/main/java/com/mrousavy/camera/NativePreviewView.kt @@ -10,8 +10,8 @@ import android.util.Size import android.view.Surface import android.view.SurfaceHolder import android.view.SurfaceView -import com.mrousavy.camera.utils.bigger -import com.mrousavy.camera.utils.smaller +import com.mrousavy.camera.extensions.bigger +import com.mrousavy.camera.extensions.smaller import kotlin.math.roundToInt /** diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraCaptureSession+capture.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt similarity index 97% rename from android/src/main/java/com/mrousavy/camera/utils/CameraCaptureSession+capture.kt rename to android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt index 7d407717e7..08663c5088 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraCaptureSession+capture.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CaptureFailure diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt similarity index 99% rename from android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt rename to android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 442fb0272a..377a13e3b0 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt similarity index 99% rename from android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt rename to android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt index edf5ab5d75..51908a825b 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDevice+createPhotoCaptureRequest.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice diff --git a/android/src/main/java/com/mrousavy/camera/utils/Context+displayRotation.kt b/android/src/main/java/com/mrousavy/camera/extensions/Context+displayRotation.kt similarity index 95% rename from android/src/main/java/com/mrousavy/camera/utils/Context+displayRotation.kt rename to android/src/main/java/com/mrousavy/camera/extensions/Context+displayRotation.kt index 92cde7404b..bb61cce028 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/Context+displayRotation.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/Context+displayRotation.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import android.content.Context import android.os.Build diff --git a/android/src/main/java/com/mrousavy/camera/utils/List+containsAny.kt b/android/src/main/java/com/mrousavy/camera/extensions/List+containsAny.kt similarity index 75% rename from android/src/main/java/com/mrousavy/camera/utils/List+containsAny.kt rename to android/src/main/java/com/mrousavy/camera/extensions/List+containsAny.kt index 17a2f9f15d..8968d1d1a3 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/List+containsAny.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/List+containsAny.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions fun List.containsAny(elements: List): Boolean { return elements.any { element -> this.contains(element) } diff --git a/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt b/android/src/main/java/com/mrousavy/camera/extensions/Size+Extensions.kt similarity index 96% rename from android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt rename to android/src/main/java/com/mrousavy/camera/extensions/Size+Extensions.kt index 6d634242c0..06ff2eeabe 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/Size+Extensions.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/Size+Extensions.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import android.util.Size import android.util.SizeF diff --git a/android/src/main/java/com/mrousavy/camera/utils/ViewGroup+installHierarchyFitter.kt b/android/src/main/java/com/mrousavy/camera/extensions/ViewGroup+installHierarchyFitter.kt similarity index 95% rename from android/src/main/java/com/mrousavy/camera/utils/ViewGroup+installHierarchyFitter.kt rename to android/src/main/java/com/mrousavy/camera/extensions/ViewGroup+installHierarchyFitter.kt index f7cc96d07a..6e79e1a435 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/ViewGroup+installHierarchyFitter.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/ViewGroup+installHierarchyFitter.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import android.view.View import android.view.ViewGroup diff --git a/android/src/main/java/com/mrousavy/camera/utils/WritableArray+Nullables.kt b/android/src/main/java/com/mrousavy/camera/extensions/WritableArray+Nullables.kt similarity index 91% rename from android/src/main/java/com/mrousavy/camera/utils/WritableArray+Nullables.kt rename to android/src/main/java/com/mrousavy/camera/extensions/WritableArray+Nullables.kt index e573e69bae..780dffbf7a 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/WritableArray+Nullables.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/WritableArray+Nullables.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import com.facebook.react.bridge.WritableArray diff --git a/android/src/main/java/com/mrousavy/camera/utils/WritableMap+Nullables.kt b/android/src/main/java/com/mrousavy/camera/extensions/WritableMap+Nullables.kt similarity index 92% rename from android/src/main/java/com/mrousavy/camera/utils/WritableMap+Nullables.kt rename to android/src/main/java/com/mrousavy/camera/extensions/WritableMap+Nullables.kt index 3543526182..91f3d69ca9 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/WritableMap+Nullables.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/WritableMap+Nullables.kt @@ -1,4 +1,4 @@ -package com.mrousavy.camera.utils +package com.mrousavy.camera.extensions import com.facebook.react.bridge.WritableMap diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 791b10deb9..43359c3e5a 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -12,6 +12,7 @@ import android.util.Size import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap +import com.mrousavy.camera.extensions.bigger import com.mrousavy.camera.parsers.parseDigitalVideoStabilizationMode import com.mrousavy.camera.parsers.parseHardwareLevel import com.mrousavy.camera.parsers.parseImageFormat From bf059d3ff6f5d24060b10315a8b1423be324de21 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 09:27:41 +0200 Subject: [PATCH 051/180] Cleanup PhotoOutputSynchronizer --- .../main/java/com/mrousavy/camera/CameraSession.kt | 2 +- .../mrousavy/camera/utils/PhotoOutputSynchronizer.kt | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 93b8275176..f6bea7d775 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -177,7 +177,7 @@ class CameraSession(private val cameraManager: CameraManager, val timestamp = result[CaptureResult.SENSOR_TIMESTAMP]!! Log.i(TAG, "Photo capture 1/2 complete - received metadata with timestamp $timestamp") try { - val image = photoOutputSynchronizer[timestamp].await() + val image = photoOutputSynchronizer.await(timestamp) // TODO: Correctly get rotationDegrees and isMirrored val rotation = ExifUtils.computeExifOrientation(0, false) diff --git a/android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt b/android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt index 84af571e8b..1dd105ee07 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/PhotoOutputSynchronizer.kt @@ -6,15 +6,21 @@ import kotlinx.coroutines.CompletableDeferred class PhotoOutputSynchronizer { private val photoOutputQueue = HashMap>() - operator fun get(key: Long): CompletableDeferred { + private operator fun get(key: Long): CompletableDeferred { if (!photoOutputQueue.containsKey(key)) { photoOutputQueue[key] = CompletableDeferred() } return photoOutputQueue[key]!! } - fun set(key: Long, image: Image) { - this[key].complete(image) + suspend fun await(timestamp: Long): Image { + val image = this[timestamp].await() + photoOutputQueue.remove(timestamp) + return image + } + + fun set(timestamp: Long, image: Image) { + this[timestamp].complete(image) } fun clear() { From 1065ae9e06c5422822916fff17f5e41ac84aed0a Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 10:49:36 +0200 Subject: [PATCH 052/180] Try just open in suspend fun --- .../java/com/mrousavy/camera/CameraQueues.kt | 4 +- .../java/com/mrousavy/camera/CameraSession.kt | 92 ++++++------------- .../main/java/com/mrousavy/camera/Errors.kt | 4 +- .../CameraDevice+createCaptureSession.kt | 6 +- .../extensions/CameraManager+openCamera.kt | 68 ++++++++++++++ 5 files changed, 107 insertions(+), 67 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt index ea538b7603..80608897b0 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraQueues.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraQueues.kt @@ -2,6 +2,7 @@ package com.mrousavy.camera import android.os.Handler import android.os.HandlerThread +import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.android.asCoroutineDispatcher import kotlinx.coroutines.asExecutor import java.util.concurrent.Executor @@ -16,12 +17,13 @@ class CameraQueues { val handler: Handler private val thread: HandlerThread val executor: Executor + val coroutineDispatcher: CoroutineDispatcher init { thread = HandlerThread(name) thread.start() handler = Handler(thread.looper) - val coroutineDispatcher = handler.asCoroutineDispatcher(name) + coroutineDispatcher = handler.asCoroutineDispatcher(name) executor = coroutineDispatcher.asExecutor() } diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index f6bea7d775..63674e3c58 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -1,6 +1,5 @@ package com.mrousavy.camera -import android.annotation.SuppressLint import android.graphics.ImageFormat import android.hardware.camera2.CameraAccessException import android.hardware.camera2.CameraCaptureSession @@ -18,6 +17,7 @@ import android.util.Range import android.util.Size import android.view.Surface import com.mrousavy.camera.extensions.FlashMode +import com.mrousavy.camera.extensions.ImageReaderOutput import com.mrousavy.camera.extensions.OutputType import com.mrousavy.camera.extensions.QualityPrioritization import com.mrousavy.camera.extensions.SessionType @@ -26,10 +26,13 @@ import com.mrousavy.camera.extensions.capture import com.mrousavy.camera.extensions.closestToOrMax import com.mrousavy.camera.extensions.createCaptureSession import com.mrousavy.camera.extensions.createPhotoCaptureRequest +import com.mrousavy.camera.extensions.openCamera +import com.mrousavy.camera.extensions.tryClose import com.mrousavy.camera.parsers.getVideoStabilizationMode -import com.mrousavy.camera.parsers.parseCameraError import com.mrousavy.camera.utils.ExifUtils import com.mrousavy.camera.utils.PhotoOutputSynchronizer +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch import java.io.Closeable import java.util.concurrent.CancellationException @@ -113,9 +116,12 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Setting Input Device to Camera $cameraId...") this.cameraId = cameraId - openCamera(cameraId) - // cameraId changed, prepare outputs. - prepareOutputs() + CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { + openCamera(cameraId) + + // cameraId changed, prepare outputs. + prepareOutputs() + } } /** @@ -196,10 +202,6 @@ class CameraSession(private val cameraManager: CameraManager, override fun onCameraAvailable(cameraId: String) { super.onCameraAvailable(cameraId) Log.i(TAG, "Camera became available: $cameraId") - if (cameraId == this.cameraId) { - // The Camera we are trying to use just became available, open it! - openCamera(cameraId) - } } override fun onCameraUnavailable(cameraId: String) { @@ -207,8 +209,7 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Camera became un-available: $cameraId") } - @SuppressLint("MissingPermission") - private fun openCamera(cameraId: String) { + private suspend fun openCamera(cameraId: String) { if (cameraIdCurrentlyOpening == cameraId) return cameraIdCurrentlyOpening = cameraId @@ -217,57 +218,19 @@ class CameraSession(private val cameraManager: CameraManager, return } - Log.i(TAG, "Opening Camera $cameraId...") - - var didOpen = false - cameraManager.openCamera(cameraId, object: CameraDevice.StateCallback() { - // When Camera is successfully opened (called once) - override fun onOpened(camera: CameraDevice) { - Log.i(TAG, "Camera $cameraId: opened!") - didOpen = true - onCameraInitialized(camera) - } - - // When Camera has been disconnected (either called on init, or later) - override fun onDisconnected(camera: CameraDevice) { - Log.i(TAG, "Camera $cameraId: disconnected!") - - if (didOpen) { - onError(CameraDisconnectedError(camera.id)) - } else { - onError(CameraCannotBeOpenedError(camera.id, "camera-disconnected")) - } - - onCameraDisconnected() - camera.close() - } - - // When Camera has been encountered an Error (either called on init, or later) - override fun onError(camera: CameraDevice, errorCode: Int) { - Log.i(TAG, "onError(${camera.id}) $errorCode") - - if (didOpen) { - onError(CameraDisconnectedError(camera.id)) - } else { - onError(CameraCannotBeOpenedError(camera.id, parseCameraError(errorCode))) - } + cameraDevice?.tryClose() + cameraDevice = null - onCameraDisconnected() - camera.close() + val camera = cameraManager.openCamera(cameraId, { camera, disconnectReason -> + if (cameraDevice == camera) { + cameraDevice = null } - }, CameraQueues.cameraQueue.handler) - } + cameraIdCurrentlyOpening = null + // TODO: Handle a disconnect and try to re-connect if possible? + onError(disconnectReason) + }, CameraQueues.cameraQueue) - private fun onCameraInitialized(camera: CameraDevice) { cameraDevice = camera - prepareSession() - } - - private fun onCameraDisconnected() { - cameraDevice = null - captureSession?.close() - captureSession = null - photoOutputSynchronizer.clear() } @@ -276,6 +239,12 @@ class CameraSession(private val cameraManager: CameraManager, * Call this whenever [cameraId], [photoOutput], [videoOutput], or [previewOutput] changes. */ private fun prepareOutputs() { + outputs.forEach { output -> + if (output is ImageReaderOutput) { + Log.i(TAG, "Closing ImageReader for ${output.outputType} output..") + output.imageReader.close() + } + } outputs.clear() val cameraId = cameraId ?: return @@ -308,7 +277,7 @@ class CameraSession(private val cameraManager: CameraManager, }, CameraQueues.videoQueue.handler) Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") - outputs.add(SurfaceOutput(imageReader.surface, OutputType.VIDEO)) + outputs.add(ImageReaderOutput(imageReader, OutputType.VIDEO)) } if (photoOutput != null && photoOutput.enabled) { @@ -326,7 +295,7 @@ class CameraSession(private val cameraManager: CameraManager, }, CameraQueues.cameraQueue.handler) Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") - outputs.add(SurfaceOutput(imageReader.surface, OutputType.PHOTO)) + outputs.add(ImageReaderOutput(imageReader, OutputType.PHOTO)) } if (previewOutput != null && previewOutput.enabled) { @@ -336,9 +305,6 @@ class CameraSession(private val cameraManager: CameraManager, } Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") - - // Outputs changed, re-create session - if (cameraDevice != null) prepareSession() } /** diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index caffc68dd0..a5c2b29921 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -49,8 +49,8 @@ class LowLightBoostNotContainedInFormatError : CameraError( ) class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") -class CameraCannotBeOpenedError(cameraId: String, error: String) : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") -class CameraDisconnectedError(cameraId: String) : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected!") +class CameraCannotBeOpenedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") +class CameraDisconnectedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error") class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.") diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 377a13e3b0..087d62e4e6 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -6,6 +6,7 @@ import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration +import android.media.ImageReader import android.os.Build import android.util.Log import android.view.Surface @@ -42,12 +43,15 @@ enum class OutputType { } } -data class SurfaceOutput(val surface: Surface, +open class SurfaceOutput(val surface: Surface, val outputType: OutputType, val dynamicRangeProfile: Long? = null) { val isRepeating: Boolean get() = outputType == OutputType.VIDEO || outputType == OutputType.PREVIEW || outputType == OutputType.VIDEO_AND_PREVIEW } +class ImageReaderOutput(val imageReader: ImageReader, + outputType: OutputType, + dynamicRangeProfile: Long? = null): SurfaceOutput(imageReader.surface, outputType, dynamicRangeProfile) fun supportsOutputType(characteristics: CameraCharacteristics, outputType: OutputType): Boolean { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt new file mode 100644 index 0000000000..0ef1b54233 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt @@ -0,0 +1,68 @@ +package com.mrousavy.camera.extensions + +import android.annotation.SuppressLint +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.os.Build +import android.util.Log +import com.mrousavy.camera.CameraCannotBeOpenedError +import com.mrousavy.camera.CameraDisconnectedError +import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.parsers.parseCameraError +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +private const val TAG = "CameraManager" + +@SuppressLint("MissingPermission") +suspend fun CameraManager.openCamera(cameraId: String, + onDisconnected: (camera: CameraDevice, reason: Throwable) -> Unit, + queue: CameraQueues.CameraQueue): CameraDevice { + return suspendCancellableCoroutine { continuation -> + Log.i(TAG, "Camera #$cameraId: Opening...") + + val callback = object: CameraDevice.StateCallback() { + override fun onOpened(camera: CameraDevice) { + Log.i(TAG, "Camera $cameraId: Opened!") + continuation.resume(camera) + } + + override fun onDisconnected(camera: CameraDevice) { + Log.i(TAG, "Camera #$cameraId: Disconnected!") + val errorCode = "disconnected" + if (continuation.isActive) { + continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, errorCode)) + } else { + onDisconnected(camera, CameraDisconnectedError(cameraId, errorCode)) + } + camera.tryClose() + } + + override fun onError(camera: CameraDevice, errorCode: Int) { + Log.e(TAG, "Camera #$cameraId: Error! $errorCode") + if (continuation.isActive) { + continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, parseCameraError(errorCode))) + } else { + onDisconnected(camera, CameraDisconnectedError(cameraId, parseCameraError(errorCode))) + } + camera.tryClose() + } + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + this.openCamera(cameraId, queue.executor, callback) + } else { + this.openCamera(cameraId, callback, queue.handler) + } + } +} + +fun CameraDevice.tryClose() { + try { + Log.i(TAG, "Camera #$id: Closing...") + this.close() + } catch (e: Throwable) { + Log.e(TAG, "Camera #$id: Failed to close!", e) + } +} From 7a10c7c5b981dd2eb6a36b35f16ad96ba03e91ea Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 11:35:38 +0200 Subject: [PATCH 053/180] Some synchronization fixes --- .../java/com/mrousavy/camera/CameraSession.kt | 176 +++++++++--------- .../main/java/com/mrousavy/camera/Errors.kt | 6 + .../CameraDevice+createCaptureSession.kt | 108 +++++++---- 3 files changed, 165 insertions(+), 125 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 63674e3c58..fb370a236b 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -33,6 +33,8 @@ import com.mrousavy.camera.utils.ExifUtils import com.mrousavy.camera.utils.PhotoOutputSynchronizer import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import java.io.Closeable import java.util.concurrent.CancellationException @@ -98,6 +100,7 @@ class CameraSession(private val cameraManager: CameraManager, private var captureSession: CameraCaptureSession? = null private var cameraIdCurrentlyOpening: String? = null private val photoOutputSynchronizer = PhotoOutputSynchronizer() + private val mutex = Mutex() init { cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) @@ -133,8 +136,11 @@ class CameraSession(private val cameraManager: CameraManager, this.photoOutput = photoOutput this.videoOutput = videoOutput this.previewOutput = previewOutput - // outputs changed, prepare them. - prepareOutputs() + + CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { + // outputs changed, prepare them. + prepareOutputs() + } } /** @@ -222,6 +228,7 @@ class CameraSession(private val cameraManager: CameraManager, cameraDevice = null val camera = cameraManager.openCamera(cameraId, { camera, disconnectReason -> + this.isActive = false if (cameraDevice == camera) { cameraDevice = null } @@ -231,6 +238,7 @@ class CameraSession(private val cameraManager: CameraManager, }, CameraQueues.cameraQueue) cameraDevice = camera + prepareSession() } @@ -238,80 +246,84 @@ class CameraSession(private val cameraManager: CameraManager, * Prepares the Image Reader and Surface outputs. * Call this whenever [cameraId], [photoOutput], [videoOutput], or [previewOutput] changes. */ - private fun prepareOutputs() { - outputs.forEach { output -> - if (output is ImageReaderOutput) { - Log.i(TAG, "Closing ImageReader for ${output.outputType} output..") - output.imageReader.close() - } - } - outputs.clear() - - val cameraId = cameraId ?: return - val videoOutput = videoOutput - val photoOutput = photoOutput - val previewOutput = previewOutput - - Log.i(TAG, "Preparing Outputs for Camera $cameraId...") - - val characteristics = cameraManager.getCameraCharacteristics(cameraId) - val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - - if (videoOutput != null && videoOutput.enabled) { - // Video or Frame Processor output: High resolution repeating images - val pixelFormat = ImageFormat.YUV_420_888 - val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoOutput.targetSize) - - val imageReader = ImageReader.newInstance(videoSize.width, - videoSize.height, - pixelFormat, - VIDEO_OUTPUT_BUFFER_SIZE) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() - if (image == null) { - Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") - return@setOnImageAvailableListener + private suspend fun prepareOutputs() { + mutex.withLock { + val cameraId = cameraId ?: return + val videoOutput = videoOutput + val photoOutput = photoOutput + val previewOutput = previewOutput + + val characteristics = cameraManager.getCameraCharacteristics(cameraId) + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + + outputs.forEach { output -> + if (output is ImageReaderOutput) { + Log.i(TAG, "Closing ImageReader for ${output.outputType} output..") + output.imageReader.close() } + } + outputs.clear() + + Log.i(TAG, "Preparing Outputs for Camera $cameraId...") + + if (videoOutput != null && videoOutput.enabled) { + // Video or Frame Processor output: High resolution repeating images + val pixelFormat = ImageFormat.YUV_420_888 + val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoOutput.targetSize) + + val imageReader = ImageReader.newInstance(videoSize.width, + videoSize.height, + pixelFormat, + VIDEO_OUTPUT_BUFFER_SIZE) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() + if (image == null) { + Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") + return@setOnImageAvailableListener + } - videoOutput.callback(image) - }, CameraQueues.videoQueue.handler) + videoOutput.callback(image) + }, CameraQueues.videoQueue.handler) - Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") - outputs.add(ImageReaderOutput(imageReader, OutputType.VIDEO)) - } + Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") + outputs.add(ImageReaderOutput(imageReader, OutputType.VIDEO)) + } - if (photoOutput != null && photoOutput.enabled) { - // Photo output: High quality still images - val pixelFormat = ImageFormat.JPEG - val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoOutput.targetSize) - - val imageReader = ImageReader.newInstance(photoSize.width, - photoSize.height, - pixelFormat, - PHOTO_OUTPUT_BUFFER_SIZE) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireLatestImage() - onPhotoCaptured(image) - }, CameraQueues.cameraQueue.handler) - - Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") - outputs.add(ImageReaderOutput(imageReader, OutputType.PHOTO)) - } + if (photoOutput != null && photoOutput.enabled) { + // Photo output: High quality still images + val pixelFormat = ImageFormat.JPEG + val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoOutput.targetSize) + + val imageReader = ImageReader.newInstance(photoSize.width, + photoSize.height, + pixelFormat, + PHOTO_OUTPUT_BUFFER_SIZE) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireLatestImage() + onPhotoCaptured(image) + }, CameraQueues.cameraQueue.handler) + + Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") + outputs.add(ImageReaderOutput(imageReader, OutputType.PHOTO)) + } + + if (previewOutput != null && previewOutput.enabled) { + // Preview output: Low resolution repeating images + Log.i(CameraView.TAG, "Adding native preview view output.") + outputs.add(SurfaceOutput(previewOutput.surface, OutputType.PREVIEW)) + } - if (previewOutput != null && previewOutput.enabled) { - // Preview output: Low resolution repeating images - Log.i(CameraView.TAG, "Adding native preview view output.") - outputs.add(SurfaceOutput(previewOutput.surface, OutputType.PREVIEW)) + Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") } - Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") + prepareSession() } /** * Creates the [CameraCaptureSession]. * Call this whenever [cameraDevice] or [outputs] changes. */ - private fun prepareSession() { + private suspend fun prepareSession() { val camera = cameraDevice ?: return if (outputs.isEmpty()) return @@ -321,35 +333,19 @@ class CameraSession(private val cameraManager: CameraManager, photoOutputSynchronizer.clear() try { - camera.createCaptureSession( - cameraManager, - SessionType.REGULAR, - outputs, - object : CameraCaptureSession.StateCallback() { - override fun onConfigured(session: CameraCaptureSession) { - Log.d(TAG, "$session Successfully configured Capture Session for Camera ${camera.id}") - captureSession = session - if (isActive) startRunning() - } + // Start up the Capture Session on the Camera + val session = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, { session -> + if (captureSession == session) { + captureSession?.close() + captureSession = null + } + isActive = false + Log.i(TAG, "Camera Session closed.") + }, CameraQueues.cameraQueue) - override fun onConfigureFailed(session: CameraCaptureSession) { - Log.d(TAG, "$session Failed to configure Capture Session for Camera ${camera.id}!") - onError(CameraCannotBeOpenedError(camera.id, "session-configuration-failed")) - } + captureSession = session - override fun onClosed(session: CameraCaptureSession) { - super.onClosed(session) - Log.d(TAG, "$session Capture Session for Camera ${camera.id} closed!") - try { - session.close() - } catch (_: Throwable) { - } - captureSession = null - photoOutputSynchronizer.clear() - } - }, - CameraQueues.cameraQueue - ) + if (isActive) startRunning() } catch (e: CameraAccessException) { Log.e(TAG, "Camera Access Exception!", e) onError(CameraCannotBeOpenedError(camera.id, "camera-not-connected-anymore")) diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index a5c2b29921..e8920cabcb 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -1,5 +1,9 @@ package com.mrousavy.camera +import com.mrousavy.camera.extensions.ImageReaderOutput +import com.mrousavy.camera.extensions.SurfaceOutput +import com.mrousavy.camera.extensions.outputsToString + abstract class CameraError( /** * The domain of the error. Error domains are used to group errors. @@ -50,6 +54,7 @@ class LowLightBoostNotContainedInFormatError : CameraError( class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") class CameraCannotBeOpenedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") +class CameraSessionCannotBeConfiguredError(cameraId: String, outputs: List) : CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera #$cameraId! Outputs: ${outputsToString(outputs)}") class CameraDisconnectedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error") class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") @@ -105,3 +110,4 @@ class NoRecordingInProgressError : CameraError("capture", "no-recording-in-progr class ViewNotFoundError(viewId: Int) : CameraError("system", "view-not-found", "The given view (ID $viewId) was not found in the view manager.") class UnknownCameraError(cause: Throwable?) : CameraError("unknown", "unknown", cause?.message ?: "An unknown camera error occured.", cause) + diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 087d62e4e6..94b19b0ddf 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -11,7 +11,11 @@ import android.os.Build import android.util.Log import android.view.Surface import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.CameraSessionCannotBeConfiguredError import com.mrousavy.camera.parsers.parseHardwareLevel +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException enum class SessionType { REGULAR, @@ -53,6 +57,14 @@ class ImageReaderOutput(val imageReader: ImageReader, outputType: OutputType, dynamicRangeProfile: Long? = null): SurfaceOutput(imageReader.surface, outputType, dynamicRangeProfile) + +fun outputsToString(outputs: List): String { + return outputs.joinToString(separator = ", ") { output -> + if (output is ImageReaderOutput) "${output.outputType} (${output.imageReader.width}x${output.imageReader.height} @${output.imageReader.imageFormat})" + else "${output.outputType}" + } +} + fun supportsOutputType(characteristics: CameraCharacteristics, outputType: OutputType): Boolean { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { val availableUseCases = characteristics.get(CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES) @@ -67,49 +79,75 @@ fun supportsOutputType(characteristics: CameraCharacteristics, outputType: Outpu } private const val TAG = "CreateCaptureSession" +private var sessionId = 1000 + +suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, + sessionType: SessionType, + outputs: List, + onClosed: (session: CameraCaptureSession) -> Unit, + queue: CameraQueues.CameraQueue): CameraCaptureSession { + return suspendCancellableCoroutine { continuation -> + val characteristics = cameraManager.getCameraCharacteristics(id) + val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! + val sessionId = sessionId++ + Log.i(TAG, "Camera $id: Creating Capture Session #$sessionId... " + + "Hardware Level: ${parseHardwareLevel(hardwareLevel)} | Outputs: ${outputsToString(outputs)}") + + val callback = object: CameraCaptureSession.StateCallback() { + override fun onConfigured(session: CameraCaptureSession) { + Log.i(TAG, "Camera $id: Opening Capture Session #$sessionId...") + continuation.resume(session) + } -fun CameraDevice.createCaptureSession(cameraManager: CameraManager, - sessionType: SessionType, - outputs: List, - callback: CameraCaptureSession.StateCallback, - queue: CameraQueues.CameraQueue) { - val characteristics = cameraManager.getCameraCharacteristics(this.id) - val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! - Log.i(TAG, "Creating Capture Session.. Hardware Level: ${parseHardwareLevel(hardwareLevel)}") - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - // API >= 24 - val outputConfigurations = arrayListOf() - for (output in outputs) { - if (!output.surface.isValid) { - Log.w(TAG, "Tried to add ${output.outputType} output, but Surface was invalid! Skipping this output..") - continue + override fun onConfigureFailed(session: CameraCaptureSession) { + Log.e(TAG, "Camera $id: Failed to configure Capture Session #$sessionId!") + continuation.resumeWithException(CameraSessionCannotBeConfiguredError(id, outputs)) } - val result = OutputConfiguration(output.surface) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - if (output.dynamicRangeProfile != null) { - result.dynamicRangeProfile = output.dynamicRangeProfile - Log.i(TAG, "Using dynamic range profile ${result.dynamicRangeProfile} for ${output.outputType} output.") + override fun onClosed(session: CameraCaptureSession) { + super.onClosed(session) + Log.i(TAG, "Camera $id: Capture Session #$sessionId closed!") + onClosed(session) + } + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + // API >= 24 + val outputConfigurations = arrayListOf() + for (output in outputs) { + if (!output.surface.isValid) { + Log.w(TAG, "Tried to add ${output.outputType} output, but Surface was invalid! Skipping this output..") + continue } - if (supportsOutputType(characteristics, output.outputType)) { - result.streamUseCase = output.outputType.toOutputType() - Log.i(TAG, "Using optimized stream use case ${result.streamUseCase} for ${output.outputType} output.") + val result = OutputConfiguration(output.surface) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + if (output.dynamicRangeProfile != null) { + result.dynamicRangeProfile = output.dynamicRangeProfile + Log.i(TAG, "Using dynamic range profile ${result.dynamicRangeProfile} for ${output.outputType} output.") + } + if (supportsOutputType(characteristics, output.outputType)) { + result.streamUseCase = output.outputType.toOutputType() + Log.i(TAG, "Using optimized stream use case ${result.streamUseCase} for ${output.outputType} output.") + } } + outputConfigurations.add(result) } - outputConfigurations.add(result) - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - // API >=28 - val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) - this.createCaptureSession(config) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + // API >=28 + Log.i(TAG, "Using new API (>=28)") + val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) + this.createCaptureSession(config) + } else { + // API >=24 + Log.i(TAG, "Using legacy API (<28)") + this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) + } } else { - // API >=24 - this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) + // API <24 + Log.i(TAG, "Using legacy API (<24)") + this.createCaptureSession(outputs.map { it.surface }, callback, queue.handler) } - } else { - // API <24 - this.createCaptureSession(outputs.map { it.surface }, callback, queue.handler) } } From 166637592b46df9aa7c2ea2a3950c53fb5c67dd4 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 11:36:49 +0200 Subject: [PATCH 054/180] fix logs --- android/src/main/java/com/mrousavy/camera/Errors.kt | 2 +- .../camera/extensions/CameraManager+openCamera.kt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index e8920cabcb..0f360a9d54 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -54,7 +54,7 @@ class LowLightBoostNotContainedInFormatError : CameraError( class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") class CameraCannotBeOpenedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") -class CameraSessionCannotBeConfiguredError(cameraId: String, outputs: List) : CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera #$cameraId! Outputs: ${outputsToString(outputs)}") +class CameraSessionCannotBeConfiguredError(cameraId: String, outputs: List) : CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera $cameraId! Outputs: ${outputsToString(outputs)}") class CameraDisconnectedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error") class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt index 0ef1b54233..d838ace777 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt @@ -20,7 +20,7 @@ suspend fun CameraManager.openCamera(cameraId: String, onDisconnected: (camera: CameraDevice, reason: Throwable) -> Unit, queue: CameraQueues.CameraQueue): CameraDevice { return suspendCancellableCoroutine { continuation -> - Log.i(TAG, "Camera #$cameraId: Opening...") + Log.i(TAG, "Camera $cameraId: Opening...") val callback = object: CameraDevice.StateCallback() { override fun onOpened(camera: CameraDevice) { @@ -29,7 +29,7 @@ suspend fun CameraManager.openCamera(cameraId: String, } override fun onDisconnected(camera: CameraDevice) { - Log.i(TAG, "Camera #$cameraId: Disconnected!") + Log.i(TAG, "Camera $cameraId: Disconnected!") val errorCode = "disconnected" if (continuation.isActive) { continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, errorCode)) @@ -40,7 +40,7 @@ suspend fun CameraManager.openCamera(cameraId: String, } override fun onError(camera: CameraDevice, errorCode: Int) { - Log.e(TAG, "Camera #$cameraId: Error! $errorCode") + Log.e(TAG, "Camera $cameraId: Error! $errorCode") if (continuation.isActive) { continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, parseCameraError(errorCode))) } else { @@ -60,9 +60,9 @@ suspend fun CameraManager.openCamera(cameraId: String, fun CameraDevice.tryClose() { try { - Log.i(TAG, "Camera #$id: Closing...") + Log.i(TAG, "Camera $id: Closing...") this.close() } catch (e: Throwable) { - Log.e(TAG, "Camera #$id: Failed to close!", e) + Log.e(TAG, "Camera $id: Failed to close!", e) } } From 92115b62ca8022913be107cca1c684939a15a917 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 11:37:39 +0200 Subject: [PATCH 055/180] Update CameraDevice+createCaptureSession.kt --- .../camera/extensions/CameraDevice+createCaptureSession.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 94b19b0ddf..5217393578 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -60,7 +60,7 @@ class ImageReaderOutput(val imageReader: ImageReader, fun outputsToString(outputs: List): String { return outputs.joinToString(separator = ", ") { output -> - if (output is ImageReaderOutput) "${output.outputType} (${output.imageReader.width}x${output.imageReader.height} @${output.imageReader.imageFormat})" + if (output is ImageReaderOutput) "${output.outputType} (${output.imageReader.width} x ${output.imageReader.height} in format #${output.imageReader.imageFormat})" else "${output.outputType}" } } From a4cecfddfcbbb3b62073d291f7b4202b42d82046 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 11:38:45 +0200 Subject: [PATCH 056/180] Update CameraDevice+createCaptureSession.kt --- .../camera/extensions/CameraDevice+createCaptureSession.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 5217393578..803df9443a 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -95,7 +95,7 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, val callback = object: CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) { - Log.i(TAG, "Camera $id: Opening Capture Session #$sessionId...") + Log.i(TAG, "Camera $id: Capture Session #$sessionId configured!") continuation.resume(session) } From e4a746676daf594fdf85fde734c0c8cf3858df3e Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 12:01:16 +0200 Subject: [PATCH 057/180] fixes --- android/src/main/java/com/mrousavy/camera/CameraView.kt | 7 +++---- .../camera/extensions/CameraDevice+createCaptureSession.kt | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 37f908f9f7..00fef29d0c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -9,16 +9,15 @@ import android.hardware.camera2.CameraManager import android.util.Log import android.util.Size import android.view.Surface -import android.view.SurfaceView import android.view.View import android.widget.FrameLayout import androidx.core.content.ContextCompat import com.facebook.react.bridge.ReadableMap -import com.mrousavy.camera.frameprocessor.Frame -import com.mrousavy.camera.frameprocessor.FrameProcessor import com.mrousavy.camera.extensions.containsAny import com.mrousavy.camera.extensions.displayRotation import com.mrousavy.camera.extensions.installHierarchyFitter +import com.mrousavy.camera.frameprocessor.Frame +import com.mrousavy.camera.frameprocessor.FrameProcessor import kotlin.math.max import kotlin.math.min @@ -151,7 +150,6 @@ class CameraView(context: Context) : FrameLayout(context) { val cameraId = cameraId ?: return if (previewType == "native") { - if (this.previewView is SurfaceView) return removeView(this.previewView) val previewView = NativePreviewView(cameraManager, cameraId, context) { surface -> @@ -167,6 +165,7 @@ class CameraView(context: Context) : FrameLayout(context) { } fun update(changedProps: ArrayList) { + Log.i(TAG, "Props changed: $changedProps") try { val shouldReconfigurePreview = changedProps.containsAny(propsThatRequirePreviewReconfiguration) val shouldReconfigureDevice = changedProps.contains("cameraId") diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 803df9443a..6b65966d54 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -59,7 +59,7 @@ class ImageReaderOutput(val imageReader: ImageReader, fun outputsToString(outputs: List): String { - return outputs.joinToString(separator = ", ") { output -> + return outputs.joinToString(", ", "[", "]") { output -> if (output is ImageReaderOutput) "${output.outputType} (${output.imageReader.width} x ${output.imageReader.height} in format #${output.imageReader.imageFormat})" else "${output.outputType}" } From 17e1bc7d080c6e8f67d2077401c7df610dba17b4 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 12:05:29 +0200 Subject: [PATCH 058/180] fix: Use Snapshot Template for speed capture prio --- .../CameraDevice+createPhotoCaptureRequest.kt | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt index 51908a825b..c38ea0bb49 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt @@ -4,49 +4,32 @@ import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest -import android.os.Build import android.view.Surface - enum class FlashMode { OFF, ON, AUTO } enum class QualityPrioritization { SPEED, BALANCED, QUALITY } -private fun supportsDeviceZsl(capabilities: IntArray): Boolean { - if (capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_PRIVATE_REPROCESSING)) return true - if (capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_YUV_REPROCESSING)) return true - return false -} - fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, surface: Surface, qualityPrioritization: QualityPrioritization, flashMode: FlashMode, enableRedEyeReduction: Boolean, enableAutoStabilization: Boolean): CaptureRequest { - val cameraCharacteristics = cameraManager.getCameraCharacteristics(this.id) - val capabilities = cameraCharacteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES)!! - val captureRequest = when (qualityPrioritization) { - // If speed, create application-specific Zero-Shutter-Lag template - QualityPrioritization.SPEED -> this.createCaptureRequest(CameraDevice.TEMPLATE_ZERO_SHUTTER_LAG) + // If speed, use snapshot template for fast capture + QualityPrioritization.SPEED -> this.createCaptureRequest(CameraDevice.TEMPLATE_VIDEO_SNAPSHOT) // Otherwise create standard still image capture template else -> this.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE) } - if (qualityPrioritization == QualityPrioritization.SPEED) { - // Some devices also support hardware Zero-Shutter-Lag, try enabling that - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && supportsDeviceZsl(capabilities)) { - captureRequest[CaptureRequest.CONTROL_ENABLE_ZSL] = true - } - } // TODO: Maybe we can even expose that prop directly? val jpegQuality = when (qualityPrioritization) { QualityPrioritization.SPEED -> 85 QualityPrioritization.BALANCED -> 92 QualityPrioritization.QUALITY -> 100 - }.toByte() - captureRequest[CaptureRequest.JPEG_QUALITY] = jpegQuality + } + captureRequest[CaptureRequest.JPEG_QUALITY] = jpegQuality.toByte() // TODO: CaptureRequest.JPEG_ORIENTATION maybe? @@ -70,6 +53,7 @@ fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, } if (enableAutoStabilization) { + val cameraCharacteristics = cameraManager.getCameraCharacteristics(this.id) // Enable optical or digital image stabilization val digitalStabilization = cameraCharacteristics.get(CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES) val hasDigitalStabilization = digitalStabilization?.contains(CameraCharacteristics.CONTROL_VIDEO_STABILIZATION_MODE_ON) ?: false From 3417d920566c59cfebd48061aceb34ded58b2d09 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 12:05:39 +0200 Subject: [PATCH 059/180] Use PREVIEW template for repeating request --- android/src/main/java/com/mrousavy/camera/CameraSession.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index fb370a236b..e5e5260b90 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -359,7 +359,7 @@ class CameraSession(private val cameraManager: CameraManager, try { Log.i(TAG, "Preparing repeating Capture Request...") - val captureRequest = captureSession.device.createCaptureRequest(CameraDevice.TEMPLATE_MANUAL) + val captureRequest = captureSession.device.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW) outputs.forEach { output -> if (output.isRepeating) { Log.i(TAG, "Adding output surface ${output.outputType}..") From 44279351c85b4cbcad689de45002093a9dd5b926 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 12:43:33 +0200 Subject: [PATCH 060/180] Use `TEMPLATE_RECORD` if video use-case is attached --- .../main/java/com/mrousavy/camera/CameraSession.kt | 6 ++++-- .../extensions/CameraDevice+createCaptureSession.kt | 11 +++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index e5e5260b90..c6c7e33020 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -359,9 +359,11 @@ class CameraSession(private val cameraManager: CameraManager, try { Log.i(TAG, "Preparing repeating Capture Request...") - val captureRequest = captureSession.device.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW) + val hasVideoOutput = outputs.any { it.outputType == OutputType.VIDEO } + val template = if (hasVideoOutput) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW + val captureRequest = captureSession.device.createCaptureRequest(template) outputs.forEach { output -> - if (output.isRepeating) { + if (output.outputType.isRepeating) { Log.i(TAG, "Adding output surface ${output.outputType}..") captureRequest.addTarget(output.surface) } diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 6b65966d54..4a4543546c 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -45,19 +45,18 @@ enum class OutputType { VIDEO_AND_PREVIEW -> 0x4 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW_VIDEO_STILL */ } } + + val isRepeating: Boolean + get() = arrayOf(VIDEO, PREVIEW, VIDEO_AND_PREVIEW).contains(this) } open class SurfaceOutput(val surface: Surface, val outputType: OutputType, - val dynamicRangeProfile: Long? = null) { - val isRepeating: Boolean - get() = outputType == OutputType.VIDEO || outputType == OutputType.PREVIEW || outputType == OutputType.VIDEO_AND_PREVIEW -} + val dynamicRangeProfile: Long? = null) class ImageReaderOutput(val imageReader: ImageReader, outputType: OutputType, dynamicRangeProfile: Long? = null): SurfaceOutput(imageReader.surface, outputType, dynamicRangeProfile) - fun outputsToString(outputs: List): String { return outputs.joinToString(", ", "[", "]") { output -> if (output is ImageReaderOutput) "${output.outputType} (${output.imageReader.width} x ${output.imageReader.height} in format #${output.imageReader.imageFormat})" @@ -65,7 +64,7 @@ fun outputsToString(outputs: List): String { } } -fun supportsOutputType(characteristics: CameraCharacteristics, outputType: OutputType): Boolean { +private fun supportsOutputType(characteristics: CameraCharacteristics, outputType: OutputType): Boolean { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { val availableUseCases = characteristics.get(CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES) if (availableUseCases != null) { From a1c1c47618347bea9a441367deaf02e3bc16cb50 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 12:46:05 +0200 Subject: [PATCH 061/180] Use `isRunning` flag --- android/src/main/java/com/mrousavy/camera/CameraSession.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index c6c7e33020..7dea0009df 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -101,6 +101,7 @@ class CameraSession(private val cameraManager: CameraManager, private var cameraIdCurrentlyOpening: String? = null private val photoOutputSynchronizer = PhotoOutputSynchronizer() private val mutex = Mutex() + private var isRunning = false init { cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) @@ -161,7 +162,7 @@ class CameraSession(private val cameraManager: CameraManager, */ fun setIsActive(isActive: Boolean) { Log.i(TAG, "setIsActive($isActive)") - if (this.isActive == isActive) { + if (isRunning == isActive) { // We're already active/inactive. return } @@ -232,6 +233,7 @@ class CameraSession(private val cameraManager: CameraManager, if (cameraDevice == camera) { cameraDevice = null } + isRunning = false cameraIdCurrentlyOpening = null // TODO: Handle a disconnect and try to re-connect if possible? onError(disconnectReason) @@ -339,7 +341,7 @@ class CameraSession(private val cameraManager: CameraManager, captureSession?.close() captureSession = null } - isActive = false + isRunning = false Log.i(TAG, "Camera Session closed.") }, CameraQueues.cameraQueue) @@ -389,6 +391,7 @@ class CameraSession(private val cameraManager: CameraManager, // Start all repeating requests (Video, Frame Processor, Preview) captureSession.setRepeatingRequest(captureRequest.build(), null, null) Log.i(TAG, "Camera Session started!") + isRunning = true onInitialized() } catch (e: IllegalStateException) { From a2f70247f27c2783c54992dae938eaa1bea143d8 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 14:13:44 +0200 Subject: [PATCH 062/180] Recreate session everytime on active/inactive --- .../java/com/mrousavy/camera/CameraSession.kt | 343 +++++++++--------- 1 file changed, 163 insertions(+), 180 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 7dea0009df..b4121482f0 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -1,7 +1,6 @@ package com.mrousavy.camera import android.graphics.ImageFormat -import android.hardware.camera2.CameraAccessException import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice @@ -95,10 +94,9 @@ class CameraSession(private val cameraManager: CameraManager, private var lowLightBoost: Boolean? = null private var hdr: Boolean? = null - private val outputs = arrayListOf() - private var cameraDevice: CameraDevice? = null private var captureSession: CameraCaptureSession? = null - private var cameraIdCurrentlyOpening: String? = null + private var cameraDevice: CameraDevice? = null + private var outputs = ArrayList() private val photoOutputSynchronizer = PhotoOutputSynchronizer() private val mutex = Mutex() private var isRunning = false @@ -109,7 +107,7 @@ class CameraSession(private val cameraManager: CameraManager, override fun close() { cameraManager.unregisterAvailabilityCallback(this) - captureSession?.close() + destroyAsync() } /** @@ -119,13 +117,7 @@ class CameraSession(private val cameraManager: CameraManager, fun setInputDevice(cameraId: String) { Log.i(TAG, "Setting Input Device to Camera $cameraId...") this.cameraId = cameraId - - CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { - openCamera(cameraId) - - // cameraId changed, prepare outputs. - prepareOutputs() - } + checkActive() } /** @@ -134,14 +126,11 @@ class CameraSession(private val cameraManager: CameraManager, fun setOutputs(photoOutput: PhotoOutput? = null, videoOutput: VideoOutput? = null, previewOutput: PreviewOutput? = null) { + Log.i(TAG, "Setting Outputs...") this.photoOutput = photoOutput this.videoOutput = videoOutput this.previewOutput = previewOutput - - CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { - // outputs changed, prepare them. - prepareOutputs() - } + checkActive() } /** @@ -151,25 +140,34 @@ class CameraSession(private val cameraManager: CameraManager, videoStabilizationMode: String? = null, hdr: Boolean? = null, lowLightBoost: Boolean? = null) { + Log.i(TAG, "Setting Format (fps: $fps | videoStabilization: $videoStabilizationMode | hdr: $hdr | lowLightBoost: $lowLightBoost)...") this.fps = fps this.videoStabilizationMode = videoStabilizationMode this.hdr = hdr this.lowLightBoost = lowLightBoost + checkActive() } /** * Starts or stops the Camera. */ fun setIsActive(isActive: Boolean) { - Log.i(TAG, "setIsActive($isActive)") - if (isRunning == isActive) { - // We're already active/inactive. - return - } - + Log.i(TAG, "Setting isActive: $isActive") this.isActive = isActive - if (isActive) startRunning() - else stopRunning() + checkActive() + } + + private fun checkActive() { + Log.i(TAG, "checkActive() isActive: $isActive | isRunning: $isRunning") + if (isActive == isRunning) return + + CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { + if (isActive) { + startRunning() + } else { + stopRunning() + } + } } suspend fun takePhoto(qualityPrioritization: QualityPrioritization, @@ -216,48 +214,14 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Camera became un-available: $cameraId") } - private suspend fun openCamera(cameraId: String) { - if (cameraIdCurrentlyOpening == cameraId) return - cameraIdCurrentlyOpening = cameraId - - if (captureSession?.device?.id == cameraId) { - Log.i(TAG, "Tried to open Camera $cameraId, but we already have a Capture Session running with that Camera. Skipping...") - return - } - - cameraDevice?.tryClose() - cameraDevice = null - - val camera = cameraManager.openCamera(cameraId, { camera, disconnectReason -> - this.isActive = false - if (cameraDevice == camera) { - cameraDevice = null - } - isRunning = false - cameraIdCurrentlyOpening = null - // TODO: Handle a disconnect and try to re-connect if possible? - onError(disconnectReason) - }, CameraQueues.cameraQueue) - - cameraDevice = camera - prepareSession() - } - - - /** - * Prepares the Image Reader and Surface outputs. - * Call this whenever [cameraId], [photoOutput], [videoOutput], or [previewOutput] changes. - */ - private suspend fun prepareOutputs() { + private suspend fun destroy() { mutex.withLock { - val cameraId = cameraId ?: return - val videoOutput = videoOutput - val photoOutput = photoOutput - val previewOutput = previewOutput - - val characteristics = cameraManager.getCameraCharacteristics(cameraId) - val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - + Log.i(TAG, "Destryoing Session...") + isRunning = false + cameraDevice?.tryClose() + cameraDevice = null + captureSession?.close() + captureSession = null outputs.forEach { output -> if (output is ImageReaderOutput) { Log.i(TAG, "Closing ImageReader for ${output.outputType} output..") @@ -265,146 +229,165 @@ class CameraSession(private val cameraManager: CameraManager, } } outputs.clear() + photoOutputSynchronizer.clear() + } + } + private fun destroyAsync() { + CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { + destroy() + } + } - Log.i(TAG, "Preparing Outputs for Camera $cameraId...") - - if (videoOutput != null && videoOutput.enabled) { - // Video or Frame Processor output: High resolution repeating images - val pixelFormat = ImageFormat.YUV_420_888 - val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoOutput.targetSize) - - val imageReader = ImageReader.newInstance(videoSize.width, - videoSize.height, - pixelFormat, - VIDEO_OUTPUT_BUFFER_SIZE) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() - if (image == null) { - Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") - return@setOnImageAvailableListener - } - - videoOutput.callback(image) - }, CameraQueues.videoQueue.handler) - - Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") - outputs.add(ImageReaderOutput(imageReader, OutputType.VIDEO)) - } - - if (photoOutput != null && photoOutput.enabled) { - // Photo output: High quality still images - val pixelFormat = ImageFormat.JPEG - val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoOutput.targetSize) - - val imageReader = ImageReader.newInstance(photoSize.width, - photoSize.height, - pixelFormat, - PHOTO_OUTPUT_BUFFER_SIZE) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireLatestImage() - onPhotoCaptured(image) - }, CameraQueues.cameraQueue.handler) - - Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") - outputs.add(ImageReaderOutput(imageReader, OutputType.PHOTO)) - } - if (previewOutput != null && previewOutput.enabled) { - // Preview output: Low resolution repeating images - Log.i(CameraView.TAG, "Adding native preview view output.") - outputs.add(SurfaceOutput(previewOutput.surface, OutputType.PREVIEW)) - } + private fun prepareOutputs(cameraId: String): ArrayList { + val videoOutput = videoOutput + val photoOutput = photoOutput + val previewOutput = previewOutput - Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") - } + val characteristics = cameraManager.getCameraCharacteristics(cameraId) + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - prepareSession() - } + val outputs = arrayListOf() - /** - * Creates the [CameraCaptureSession]. - * Call this whenever [cameraDevice] or [outputs] changes. - */ - private suspend fun prepareSession() { - val camera = cameraDevice ?: return - if (outputs.isEmpty()) return + Log.i(TAG, "Preparing Outputs for Camera $cameraId...") - Log.i(TAG, "Creating CameraCaptureSession for Camera ${camera.id}...") - captureSession?.close() - captureSession = null - photoOutputSynchronizer.clear() + if (videoOutput != null && videoOutput.enabled) { + // Video or Frame Processor output: High resolution repeating images + val pixelFormat = ImageFormat.YUV_420_888 + val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoOutput.targetSize) - try { - // Start up the Capture Session on the Camera - val session = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, { session -> - if (captureSession == session) { - captureSession?.close() - captureSession = null + val imageReader = ImageReader.newInstance(videoSize.width, + videoSize.height, + pixelFormat, + VIDEO_OUTPUT_BUFFER_SIZE) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() + if (image == null) { + Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") + return@setOnImageAvailableListener } - isRunning = false - Log.i(TAG, "Camera Session closed.") - }, CameraQueues.cameraQueue) - captureSession = session + videoOutput.callback(image) + }, CameraQueues.videoQueue.handler) - if (isActive) startRunning() - } catch (e: CameraAccessException) { - Log.e(TAG, "Camera Access Exception!", e) - onError(CameraCannotBeOpenedError(camera.id, "camera-not-connected-anymore")) + Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") + outputs.add(ImageReaderOutput(imageReader, OutputType.VIDEO)) } - } - private fun startRunning() { - val captureSession = captureSession ?: return + if (photoOutput != null && photoOutput.enabled) { + // Photo output: High quality still images + val pixelFormat = ImageFormat.JPEG + val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoOutput.targetSize) + + val imageReader = ImageReader.newInstance(photoSize.width, + photoSize.height, + pixelFormat, + PHOTO_OUTPUT_BUFFER_SIZE) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireLatestImage() + onPhotoCaptured(image) + }, CameraQueues.cameraQueue.handler) + + Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") + outputs.add(ImageReaderOutput(imageReader, OutputType.PHOTO)) + } - Log.i(TAG, "Starting Camera Session...") - try { - Log.i(TAG, "Preparing repeating Capture Request...") + if (previewOutput != null && previewOutput.enabled) { + // Preview output: Low resolution repeating images + Log.i(CameraView.TAG, "Adding native preview view output.") + outputs.add(SurfaceOutput(previewOutput.surface, OutputType.PREVIEW)) + } - val hasVideoOutput = outputs.any { it.outputType == OutputType.VIDEO } - val template = if (hasVideoOutput) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW - val captureRequest = captureSession.device.createCaptureRequest(template) - outputs.forEach { output -> - if (output.outputType.isRepeating) { - Log.i(TAG, "Adding output surface ${output.outputType}..") - captureRequest.addTarget(output.surface) - } - } + Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") - fps?.let { fps -> - captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) - } - videoStabilizationMode?.let { videoStabilizationMode -> - val stabilizationMode = getVideoStabilizationMode(videoStabilizationMode) - captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, stabilizationMode.digitalMode) - captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, stabilizationMode.opticalMode) - } - if (lowLightBoost == true) { - captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) + return outputs + } + + private fun preparePreviewCaptureRequest(captureSession: CameraCaptureSession, + outputs: List): CaptureRequest { + val hasVideoOutput = outputs.any { it.outputType == OutputType.VIDEO } + val template = if (hasVideoOutput) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW + val captureRequest = captureSession.device.createCaptureRequest(template) + outputs.forEach { output -> + if (output.outputType.isRepeating) { + Log.i(TAG, "Adding output surface ${output.outputType}..") + captureRequest.addTarget(output.surface) } - if (hdr == true) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP_MR1) { - captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR) - } + } + + fps?.let { fps -> + captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) + } + videoStabilizationMode?.let { videoStabilizationMode -> + val stabilizationMode = getVideoStabilizationMode(videoStabilizationMode) + captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, stabilizationMode.digitalMode) + captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, stabilizationMode.opticalMode) + } + if (lowLightBoost == true) { + captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) + } + if (hdr == true) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP_MR1) { + captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR) } + } + return captureRequest.build() + } - // Start all repeating requests (Video, Frame Processor, Preview) - captureSession.setRepeatingRequest(captureRequest.build(), null, null) - Log.i(TAG, "Camera Session started!") - isRunning = true + private suspend fun startRunning() { + val cameraId = cameraId ?: return - onInitialized() + // 0. Delete everything we have right now + destroy() + + Log.i(TAG, "Starting Camera Session...") + + try { + mutex.withLock { + // 1. Create outputs for device (PREVIEW, PHOTO, VIDEO) + val outputs = prepareOutputs(cameraId) + if (outputs.isEmpty()) return + + // 2. Open Camera Device + val camera = cameraManager.openCamera(cameraId, { camera, reason -> + isRunning = false + if (cameraDevice == camera) destroyAsync() + onError(reason) + }, CameraQueues.cameraQueue) + + // 3. Create capture session with outputs + val session = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, { session -> + isRunning = false + if (captureSession == session) destroyAsync() + }, CameraQueues.cameraQueue) + + // 4. Create repeating request (configures FPS, HDR, etc.) + val repeatingRequest = preparePreviewCaptureRequest(session, outputs) + + // 5. Start repeating request + session.setRepeatingRequest(repeatingRequest, null, null) + + Log.i(TAG, "Camera Session started!") + isRunning = true + this.captureSession = session + this.outputs = outputs + this.cameraDevice = camera + + onInitialized() + } } catch (e: IllegalStateException) { Log.w(TAG, "Failed to start Camera Session, this session is already closed.") } } - private fun stopRunning() { + private suspend fun stopRunning() { Log.i(TAG, "Stopping Camera Session...") try { - val captureSession = captureSession ?: return - captureSession.stopRepeating() - Log.i(TAG, "Camera Session stopped!") + mutex.withLock { + val captureSession = captureSession ?: return + captureSession.stopRepeating() + Log.i(TAG, "Camera Session stopped!") + } } catch (e: IllegalStateException) { Log.w(TAG, "Failed to stop Camera Session, this session is already closed.") } From 90010020cbb14c98572f0c48cd462201c07cc8cf Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 14:35:14 +0200 Subject: [PATCH 063/180] Lazily get values in capture session --- .../java/com/mrousavy/camera/CameraSession.kt | 118 ++++++++++++------ 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index b4121482f0..7e8ff9271c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -107,7 +107,11 @@ class CameraSession(private val cameraManager: CameraManager, override fun close() { cameraManager.unregisterAvailabilityCallback(this) - destroyAsync() + photoOutputSynchronizer.clear() + captureSession?.close() + cameraDevice?.tryClose() + outputs.clear() + isRunning = false } /** @@ -214,32 +218,53 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Camera became un-available: $cameraId") } - private suspend fun destroy() { - mutex.withLock { - Log.i(TAG, "Destryoing Session...") - isRunning = false - cameraDevice?.tryClose() - cameraDevice = null - captureSession?.close() - captureSession = null - outputs.forEach { output -> - if (output is ImageReaderOutput) { - Log.i(TAG, "Closing ImageReader for ${output.outputType} output..") - output.imageReader.close() - } + private fun destroy() { + Log.i(TAG, "Destroying Session...") + isRunning = false + cameraDevice?.tryClose() + cameraDevice = null + captureSession?.close() + captureSession = null + outputs.forEach { output -> + if (output is ImageReaderOutput) { + Log.i(TAG, "Closing ImageReader for ${output.outputType} output..") + output.imageReader.close() } - outputs.clear() - photoOutputSynchronizer.clear() } + outputs.clear() + photoOutputSynchronizer.clear() + Log.i(TAG, "Session destroyed!") } - private fun destroyAsync() { - CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { - destroy() + + /** + * Opens a [CameraDevice]. If there already is an open Camera for the given [cameraId], use that. + */ + private suspend fun getCameraDevice(cameraId: String, onClosed: (error: Throwable) -> Unit): CameraDevice { + val currentDevice = cameraDevice + if (currentDevice?.id == cameraId) { + // We already opened that device + return currentDevice } - } + // Close previous device + cameraDevice?.tryClose() + cameraDevice = null + + val device = cameraManager.openCamera(cameraId, { camera, reason -> + if (cameraDevice == camera) { + // The current CameraDevice has been closed, handle that! + onClosed(reason) + cameraDevice = null + } else { + // A new CameraDevice has been opened, we don't care about this one anymore. + } + }, CameraQueues.cameraQueue) + // Cache device in memory + cameraDevice = device + return device + } - private fun prepareOutputs(cameraId: String): ArrayList { + private fun getOutputs(cameraId: String): ArrayList { val videoOutput = videoOutput val photoOutput = photoOutput val previewOutput = previewOutput @@ -303,8 +328,34 @@ class CameraSession(private val cameraManager: CameraManager, return outputs } - private fun preparePreviewCaptureRequest(captureSession: CameraCaptureSession, - outputs: List): CaptureRequest { + private suspend fun getCaptureSession(cameraDevice: CameraDevice, outputs: List, onClosed: () -> Unit): CameraCaptureSession { + val currentSession = captureSession + // TODO: Also compare outputs!! + if (currentSession?.device == cameraDevice) { + // We already opened a CameraCaptureSession on this device + return currentSession + } + captureSession?.close() + // TODO: Call abortCaptures() as well? + captureSession = null + + val session = cameraDevice.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, { session -> + if (captureSession == session) { + // The current CameraCaptureSession has been closed, handle that! + onClosed() + captureSession = null + } else { + // A new CameraCaptureSession has been opened, we don't care about this one anymore. + } + }, CameraQueues.cameraQueue) + + // Cache session in memory + captureSession = session + return session + } + + private fun getPreviewCaptureRequest(captureSession: CameraCaptureSession, + outputs: List): CaptureRequest { val hasVideoOutput = outputs.any { it.outputType == OutputType.VIDEO } val template = if (hasVideoOutput) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW val captureRequest = captureSession.device.createCaptureRequest(template) @@ -335,34 +386,31 @@ class CameraSession(private val cameraManager: CameraManager, } private suspend fun startRunning() { + isRunning = false val cameraId = cameraId ?: return - // 0. Delete everything we have right now - destroy() - Log.i(TAG, "Starting Camera Session...") try { mutex.withLock { // 1. Create outputs for device (PREVIEW, PHOTO, VIDEO) - val outputs = prepareOutputs(cameraId) + val outputs = getOutputs(cameraId) if (outputs.isEmpty()) return // 2. Open Camera Device - val camera = cameraManager.openCamera(cameraId, { camera, reason -> + val camera = getCameraDevice(cameraId) { reason -> isRunning = false - if (cameraDevice == camera) destroyAsync() onError(reason) - }, CameraQueues.cameraQueue) + } // 3. Create capture session with outputs - val session = camera.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, { session -> - isRunning = false - if (captureSession == session) destroyAsync() - }, CameraQueues.cameraQueue) + val session = getCaptureSession(camera, outputs) { + isRunning = false + onError(CameraDisconnectedError(cameraId, "session-closed")) + } // 4. Create repeating request (configures FPS, HDR, etc.) - val repeatingRequest = preparePreviewCaptureRequest(session, outputs) + val repeatingRequest = getPreviewCaptureRequest(session, outputs) // 5. Start repeating request session.setRepeatingRequest(repeatingRequest, null, null) From 1e9db32885c9e626b59a01fc89cf0ab6241f823d Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 14:44:25 +0200 Subject: [PATCH 064/180] Stability --- .../java/com/mrousavy/camera/CameraSession.kt | 19 ++++++++++++++----- .../java/com/mrousavy/camera/CameraView.kt | 6 ++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 7e8ff9271c..56f318ce7b 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -36,6 +36,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.io.Closeable import java.util.concurrent.CancellationException +import kotlin.coroutines.CoroutineContext // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing @@ -54,7 +55,7 @@ import java.util.concurrent.CancellationException */ class CameraSession(private val cameraManager: CameraManager, private val onInitialized: () -> Unit, - private val onError: (e: Throwable) -> Unit): Closeable, CameraManager.AvailabilityCallback() { + private val onError: (e: Throwable) -> Unit): CoroutineScope, Closeable, CameraManager.AvailabilityCallback() { companion object { private const val TAG = "CameraSession" private const val PHOTO_OUTPUT_BUFFER_SIZE = 3 @@ -101,6 +102,8 @@ class CameraSession(private val cameraManager: CameraManager, private val mutex = Mutex() private var isRunning = false + override val coroutineContext: CoroutineContext = CameraQueues.cameraQueue.coroutineDispatcher + init { cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler) } @@ -121,7 +124,9 @@ class CameraSession(private val cameraManager: CameraManager, fun setInputDevice(cameraId: String) { Log.i(TAG, "Setting Input Device to Camera $cameraId...") this.cameraId = cameraId - checkActive() + launch { + startRunning() + } } /** @@ -134,7 +139,9 @@ class CameraSession(private val cameraManager: CameraManager, this.photoOutput = photoOutput this.videoOutput = videoOutput this.previewOutput = previewOutput - checkActive() + launch { + startRunning() + } } /** @@ -149,7 +156,9 @@ class CameraSession(private val cameraManager: CameraManager, this.videoStabilizationMode = videoStabilizationMode this.hdr = hdr this.lowLightBoost = lowLightBoost - checkActive() + launch { + startRunning() + } } /** @@ -165,7 +174,7 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "checkActive() isActive: $isActive | isRunning: $isRunning") if (isActive == isRunning) return - CoroutineScope(CameraQueues.cameraQueue.coroutineDispatcher).launch { + launch { if (isActive) { startRunning() } else { diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 00fef29d0c..078dae3cd6 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -222,9 +222,7 @@ class CameraView(context: Context) : FrameLayout(context) { val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null - val previewSurface = previewSurface ?: return - - if (!previewSurface.isValid) return + val previewSurface = previewSurface cameraSession.setOutputs( // Photo Output @@ -235,7 +233,7 @@ class CameraView(context: Context) : FrameLayout(context) { onFrame(frame) }, targetVideoSize), // Preview Output - CameraSession.PreviewOutput(true, previewSurface) + if (previewSurface?.isValid == true) CameraSession.PreviewOutput(true, previewSurface) else null ) } From 9f79e6a677934180fe37315d3193a87135aaf826 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 15:55:28 +0200 Subject: [PATCH 065/180] Rebuild session if outputs change --- android/src/main/java/com/mrousavy/camera/CameraSession.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 56f318ce7b..96bcd35f5b 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -100,6 +100,7 @@ class CameraSession(private val cameraManager: CameraManager, private var outputs = ArrayList() private val photoOutputSynchronizer = PhotoOutputSynchronizer() private val mutex = Mutex() + private var didOutputsChange = false private var isRunning = false override val coroutineContext: CoroutineContext = CameraQueues.cameraQueue.coroutineDispatcher @@ -139,6 +140,7 @@ class CameraSession(private val cameraManager: CameraManager, this.photoOutput = photoOutput this.videoOutput = videoOutput this.previewOutput = previewOutput + didOutputsChange = true launch { startRunning() } @@ -339,8 +341,7 @@ class CameraSession(private val cameraManager: CameraManager, private suspend fun getCaptureSession(cameraDevice: CameraDevice, outputs: List, onClosed: () -> Unit): CameraCaptureSession { val currentSession = captureSession - // TODO: Also compare outputs!! - if (currentSession?.device == cameraDevice) { + if (currentSession?.device == cameraDevice && !didOutputsChange) { // We already opened a CameraCaptureSession on this device return currentSession } From ec606e8604fbdb84e860be0286eb4ba78da21df3 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 4 Aug 2023 16:11:05 +0200 Subject: [PATCH 066/180] Set `didOutputsChange` back to false --- android/src/main/java/com/mrousavy/camera/CameraSession.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 96bcd35f5b..dc87be417c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -361,6 +361,7 @@ class CameraSession(private val cameraManager: CameraManager, // Cache session in memory captureSession = session + didOutputsChange = false return session } From 50c6f164133fd8cb8151db71615c71bfd49e9bb1 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 5 Aug 2023 10:41:56 +0200 Subject: [PATCH 067/180] Capture first in lock --- .../java/com/mrousavy/camera/CameraSession.kt | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index dc87be417c..27ba857e31 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -275,11 +275,10 @@ class CameraSession(private val cameraManager: CameraManager, return device } - private fun getOutputs(cameraId: String): ArrayList { - val videoOutput = videoOutput - val photoOutput = photoOutput - val previewOutput = previewOutput - + private fun getOutputs(cameraId: String, + videoOutput: VideoOutput? = null, + photoOutput: PhotoOutput? = null, + previewOutput: PreviewOutput? = null): ArrayList { val characteristics = cameraManager.getCameraCharacteristics(cameraId) val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! @@ -339,7 +338,9 @@ class CameraSession(private val cameraManager: CameraManager, return outputs } - private suspend fun getCaptureSession(cameraDevice: CameraDevice, outputs: List, onClosed: () -> Unit): CameraCaptureSession { + private suspend fun getCaptureSession(cameraDevice: CameraDevice, + outputs: List, + onClosed: () -> Unit): CameraCaptureSession { val currentSession = captureSession if (currentSession?.device == cameraDevice && !didOutputsChange) { // We already opened a CameraCaptureSession on this device @@ -366,7 +367,11 @@ class CameraSession(private val cameraManager: CameraManager, } private fun getPreviewCaptureRequest(captureSession: CameraCaptureSession, - outputs: List): CaptureRequest { + outputs: List, + fps: Int? = null, + videoStabilizationMode: String? = null, + lowLightBoost: Boolean? = null, + hdr: Boolean? = null): CaptureRequest { val hasVideoOutput = outputs.any { it.outputType == OutputType.VIDEO } val template = if (hasVideoOutput) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW val captureRequest = captureSession.device.createCaptureRequest(template) @@ -377,10 +382,11 @@ class CameraSession(private val cameraManager: CameraManager, } } - fps?.let { fps -> + + if (fps != null) { captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) } - videoStabilizationMode?.let { videoStabilizationMode -> + if (videoStabilizationMode != null) { val stabilizationMode = getVideoStabilizationMode(videoStabilizationMode) captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, stabilizationMode.digitalMode) captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, stabilizationMode.opticalMode) @@ -404,8 +410,16 @@ class CameraSession(private val cameraManager: CameraManager, try { mutex.withLock { + val videoOutput = videoOutput + val photoOutput = photoOutput + val previewOutput = previewOutput + val fps = fps + val videoStabilizationMode = videoStabilizationMode + val lowLightBoost = lowLightBoost + val hdr = hdr + // 1. Create outputs for device (PREVIEW, PHOTO, VIDEO) - val outputs = getOutputs(cameraId) + val outputs = getOutputs(cameraId, videoOutput, photoOutput, previewOutput) if (outputs.isEmpty()) return // 2. Open Camera Device @@ -421,7 +435,7 @@ class CameraSession(private val cameraManager: CameraManager, } // 4. Create repeating request (configures FPS, HDR, etc.) - val repeatingRequest = getPreviewCaptureRequest(session, outputs) + val repeatingRequest = getPreviewCaptureRequest(session, outputs, fps, videoStabilizationMode, lowLightBoost, hdr) // 5. Start repeating request session.setRepeatingRequest(repeatingRequest, null, null) From 2107f2c61be5a7346532976806a58ca5ff110604 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 5 Aug 2023 10:57:58 +0200 Subject: [PATCH 068/180] Try --- .../java/com/mrousavy/camera/CameraSession.kt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 27ba857e31..4624fe9a2c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -100,7 +100,6 @@ class CameraSession(private val cameraManager: CameraManager, private var outputs = ArrayList() private val photoOutputSynchronizer = PhotoOutputSynchronizer() private val mutex = Mutex() - private var didOutputsChange = false private var isRunning = false override val coroutineContext: CoroutineContext = CameraQueues.cameraQueue.coroutineDispatcher @@ -140,7 +139,6 @@ class CameraSession(private val cameraManager: CameraManager, this.photoOutput = photoOutput this.videoOutput = videoOutput this.previewOutput = previewOutput - didOutputsChange = true launch { startRunning() } @@ -342,7 +340,7 @@ class CameraSession(private val cameraManager: CameraManager, outputs: List, onClosed: () -> Unit): CameraCaptureSession { val currentSession = captureSession - if (currentSession?.device == cameraDevice && !didOutputsChange) { + if (currentSession?.device == cameraDevice && false) { // We already opened a CameraCaptureSession on this device return currentSession } @@ -362,7 +360,6 @@ class CameraSession(private val cameraManager: CameraManager, // Cache session in memory captureSession = session - didOutputsChange = false return session } @@ -418,6 +415,11 @@ class CameraSession(private val cameraManager: CameraManager, val lowLightBoost = lowLightBoost val hdr = hdr + if (previewOutput == null) { + cameraDevice?.tryClose() + return@withLock + } + // 1. Create outputs for device (PREVIEW, PHOTO, VIDEO) val outputs = getOutputs(cameraId, videoOutput, photoOutput, previewOutput) if (outputs.isEmpty()) return @@ -449,7 +451,7 @@ class CameraSession(private val cameraManager: CameraManager, onInitialized() } } catch (e: IllegalStateException) { - Log.w(TAG, "Failed to start Camera Session, this session is already closed.") + Log.e(TAG, "Failed to start Camera Session, this session is already closed.", e) } } @@ -457,12 +459,12 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Stopping Camera Session...") try { mutex.withLock { - val captureSession = captureSession ?: return - captureSession.stopRepeating() + val camera = cameraDevice ?: return + camera.close() Log.i(TAG, "Camera Session stopped!") } } catch (e: IllegalStateException) { - Log.w(TAG, "Failed to stop Camera Session, this session is already closed.") + Log.e(TAG, "Failed to stop Camera Session, this session is already closed.", e) } } } From a7e1e9a817cade861ebe9a71857904d4b81d3ead Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 5 Aug 2023 12:49:57 +0200 Subject: [PATCH 069/180] kinda fix it? idk --- .../java/com/mrousavy/camera/CameraSession.kt | 208 +++++------------- .../java/com/mrousavy/camera/CameraView.kt | 24 +- .../main/java/com/mrousavy/camera/Errors.kt | 6 +- .../CameraDevice+createCaptureSession.kt | 91 ++++---- .../mrousavy/camera/utils/CameraOutputs.kt | 136 ++++++++++++ 5 files changed, 254 insertions(+), 211 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 4624fe9a2c..65da39b2c7 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -1,33 +1,25 @@ package com.mrousavy.camera -import android.graphics.ImageFormat import android.hardware.camera2.CameraCaptureSession -import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult import android.hardware.camera2.TotalCaptureResult import android.media.Image -import android.media.ImageReader import android.os.Build import android.util.Log import android.util.Range -import android.util.Size -import android.view.Surface import com.mrousavy.camera.extensions.FlashMode -import com.mrousavy.camera.extensions.ImageReaderOutput -import com.mrousavy.camera.extensions.OutputType import com.mrousavy.camera.extensions.QualityPrioritization import com.mrousavy.camera.extensions.SessionType -import com.mrousavy.camera.extensions.SurfaceOutput import com.mrousavy.camera.extensions.capture -import com.mrousavy.camera.extensions.closestToOrMax import com.mrousavy.camera.extensions.createCaptureSession import com.mrousavy.camera.extensions.createPhotoCaptureRequest import com.mrousavy.camera.extensions.openCamera import com.mrousavy.camera.extensions.tryClose import com.mrousavy.camera.parsers.getVideoStabilizationMode +import com.mrousavy.camera.utils.CameraOutputs import com.mrousavy.camera.utils.ExifUtils import com.mrousavy.camera.utils.PhotoOutputSynchronizer import kotlinx.coroutines.CoroutineScope @@ -40,34 +32,12 @@ import kotlin.coroutines.CoroutineContext // TODO: Use reprocessable YUV capture session for more efficient Skia Frame Processing -/** - * A Camera Session. - * Flow: - * - * 1. [cameraDevice] gets rebuilt everytime [cameraId] changes - * 2. [outputs] get rebuilt everytime [photoOutput], [videoOutput], [previewOutput] or [cameraDevice] changes. - * 3. [captureSession] gets rebuilt everytime [outputs] changes. - * 4. [startRunning]/[stopRunning] gets called everytime [isActive] or [captureSession] changes. - * - * Examples: - * - Changing [cameraId] causes everything to be rebuilt. - * - Changing [videoOutput] causes all [outputs] to be rebuilt, which later causes the [captureSession] to be rebuilt. - */ class CameraSession(private val cameraManager: CameraManager, private val onInitialized: () -> Unit, - private val onError: (e: Throwable) -> Unit): CoroutineScope, Closeable, CameraManager.AvailabilityCallback() { + private val onError: (e: Throwable) -> Unit): CoroutineScope, Closeable, CameraOutputs.Callback, CameraManager.AvailabilityCallback() { companion object { private const val TAG = "CameraSession" - private const val PHOTO_OUTPUT_BUFFER_SIZE = 3 - private const val VIDEO_OUTPUT_BUFFER_SIZE = 2 } - data class VideoOutput(val enabled: Boolean, - val callback: (image: Image) -> Unit, - val targetSize: Size? = null) - data class PhotoOutput(val enabled: Boolean, - val targetSize: Size? = null) - data class PreviewOutput(val enabled: Boolean, - val surface: Surface) data class CapturedPhoto(val image: Image, val metadata: TotalCaptureResult, @@ -82,9 +52,7 @@ class CameraSession(private val cameraManager: CameraManager, private var cameraId: String? = null // setOutputs(..) - private var photoOutput: PhotoOutput? = null - private var videoOutput: VideoOutput? = null - private var previewOutput: PreviewOutput? = null + private var outputs: CameraOutputs? = null // setIsActive(..) private var isActive = false @@ -97,7 +65,6 @@ class CameraSession(private val cameraManager: CameraManager, private var captureSession: CameraCaptureSession? = null private var cameraDevice: CameraDevice? = null - private var outputs = ArrayList() private val photoOutputSynchronizer = PhotoOutputSynchronizer() private val mutex = Mutex() private var isRunning = false @@ -113,7 +80,7 @@ class CameraSession(private val cameraManager: CameraManager, photoOutputSynchronizer.clear() captureSession?.close() cameraDevice?.tryClose() - outputs.clear() + outputs?.close() isRunning = false } @@ -132,13 +99,23 @@ class CameraSession(private val cameraManager: CameraManager, /** * Configure the outputs of the Camera. */ - fun setOutputs(photoOutput: PhotoOutput? = null, - videoOutput: VideoOutput? = null, - previewOutput: PreviewOutput? = null) { + fun setOutputs(preview: CameraOutputs.PreviewOutput? = null, + photo: CameraOutputs.PhotoOutput? = null, + video: CameraOutputs.VideoOutput? = null) { Log.i(TAG, "Setting Outputs...") - this.photoOutput = photoOutput - this.videoOutput = videoOutput - this.previewOutput = previewOutput + val cameraId = cameraId ?: throw NoCameraDeviceError() + val newOutputs = CameraOutputs(cameraId, + cameraManager, + preview, + photo, + video, + this) + if (this.outputs == newOutputs) { + Log.i(TAG, "Outputs didn't change, skipping restart..") + return + } + + this.outputs = newOutputs launch { startRunning() } @@ -165,13 +142,8 @@ class CameraSession(private val cameraManager: CameraManager, * Starts or stops the Camera. */ fun setIsActive(isActive: Boolean) { - Log.i(TAG, "Setting isActive: $isActive") + Log.i(TAG, "Setting isActive: $isActive (isRunning: $isRunning)") this.isActive = isActive - checkActive() - } - - private fun checkActive() { - Log.i(TAG, "checkActive() isActive: $isActive | isRunning: $isRunning") if (isActive == isRunning) return launch { @@ -188,8 +160,9 @@ class CameraSession(private val cameraManager: CameraManager, enableRedEyeReduction: Boolean, enableAutoStabilization: Boolean): CapturedPhoto { val captureSession = captureSession ?: throw CameraNotReadyError() + val outputs = outputs ?: throw CameraNotReadyError() - val photoOutput = outputs.find { it.outputType == OutputType.PHOTO } ?: throw PhotoNotEnabledError() + val photoOutput = outputs.photoOutput ?: throw PhotoNotEnabledError() val captureRequest = captureSession.device.createPhotoCaptureRequest(cameraManager, photoOutput.surface, qualityPrioritization, @@ -212,7 +185,7 @@ class CameraSession(private val cameraManager: CameraManager, } } - private fun onPhotoCaptured(image: Image) { + override fun onPhotoCaptured(image: Image) { Log.i(CameraView.TAG, "Photo captured! ${image.width} x ${image.height}") photoOutputSynchronizer.set(image.timestamp, image) } @@ -227,24 +200,6 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Camera became un-available: $cameraId") } - private fun destroy() { - Log.i(TAG, "Destroying Session...") - isRunning = false - cameraDevice?.tryClose() - cameraDevice = null - captureSession?.close() - captureSession = null - outputs.forEach { output -> - if (output is ImageReaderOutput) { - Log.i(TAG, "Closing ImageReader for ${output.outputType} output..") - output.imageReader.close() - } - } - outputs.clear() - photoOutputSynchronizer.clear() - Log.i(TAG, "Session destroyed!") - } - /** * Opens a [CameraDevice]. If there already is an open Camera for the given [cameraId], use that. */ @@ -259,6 +214,7 @@ class CameraSession(private val cameraManager: CameraManager, cameraDevice = null val device = cameraManager.openCamera(cameraId, { camera, reason -> + Log.d(TAG, "Camera Closed ($cameraDevice == $camera)") if (cameraDevice == camera) { // The current CameraDevice has been closed, handle that! onClosed(reason) @@ -273,74 +229,12 @@ class CameraSession(private val cameraManager: CameraManager, return device } - private fun getOutputs(cameraId: String, - videoOutput: VideoOutput? = null, - photoOutput: PhotoOutput? = null, - previewOutput: PreviewOutput? = null): ArrayList { - val characteristics = cameraManager.getCameraCharacteristics(cameraId) - val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - - val outputs = arrayListOf() - - Log.i(TAG, "Preparing Outputs for Camera $cameraId...") - - if (videoOutput != null && videoOutput.enabled) { - // Video or Frame Processor output: High resolution repeating images - val pixelFormat = ImageFormat.YUV_420_888 - val videoSize = config.getOutputSizes(pixelFormat).closestToOrMax(videoOutput.targetSize) - - val imageReader = ImageReader.newInstance(videoSize.width, - videoSize.height, - pixelFormat, - VIDEO_OUTPUT_BUFFER_SIZE) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() - if (image == null) { - Log.w(CameraView.TAG, "Failed to get new Image from ImageReader, dropping a Frame...") - return@setOnImageAvailableListener - } - - videoOutput.callback(image) - }, CameraQueues.videoQueue.handler) - - Log.i(CameraView.TAG, "Adding ${videoSize.width}x${videoSize.height} video output. (Format: $pixelFormat)") - outputs.add(ImageReaderOutput(imageReader, OutputType.VIDEO)) - } - - if (photoOutput != null && photoOutput.enabled) { - // Photo output: High quality still images - val pixelFormat = ImageFormat.JPEG - val photoSize = config.getOutputSizes(pixelFormat).closestToOrMax(photoOutput.targetSize) - - val imageReader = ImageReader.newInstance(photoSize.width, - photoSize.height, - pixelFormat, - PHOTO_OUTPUT_BUFFER_SIZE) - imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireLatestImage() - onPhotoCaptured(image) - }, CameraQueues.cameraQueue.handler) - - Log.i(CameraView.TAG, "Adding ${photoSize.width}x${photoSize.height} photo output. (Format: $pixelFormat)") - outputs.add(ImageReaderOutput(imageReader, OutputType.PHOTO)) - } - - if (previewOutput != null && previewOutput.enabled) { - // Preview output: Low resolution repeating images - Log.i(CameraView.TAG, "Adding native preview view output.") - outputs.add(SurfaceOutput(previewOutput.surface, OutputType.PREVIEW)) - } - - Log.i(TAG, "Prepared ${outputs.size} Outputs for Camera $cameraId!") - - return outputs - } - private suspend fun getCaptureSession(cameraDevice: CameraDevice, - outputs: List, + outputs: CameraOutputs, onClosed: () -> Unit): CameraCaptureSession { val currentSession = captureSession - if (currentSession?.device == cameraDevice && false) { + // TODO: Compare if outputs changed... Attach outputs to CameraCaptureSession? + if (currentSession?.device == cameraDevice && this.outputs == outputs) { // We already opened a CameraCaptureSession on this device return currentSession } @@ -349,6 +243,7 @@ class CameraSession(private val cameraManager: CameraManager, captureSession = null val session = cameraDevice.createCaptureSession(cameraManager, SessionType.REGULAR, outputs, { session -> + Log.d(TAG, "Capture Session Closed ($captureSession == $session)") if (captureSession == session) { // The current CameraCaptureSession has been closed, handle that! onClosed() @@ -359,26 +254,27 @@ class CameraSession(private val cameraManager: CameraManager, }, CameraQueues.cameraQueue) // Cache session in memory + this.outputs = outputs captureSession = session return session } private fun getPreviewCaptureRequest(captureSession: CameraCaptureSession, - outputs: List, + outputs: CameraOutputs, fps: Int? = null, videoStabilizationMode: String? = null, lowLightBoost: Boolean? = null, hdr: Boolean? = null): CaptureRequest { - val hasVideoOutput = outputs.any { it.outputType == OutputType.VIDEO } - val template = if (hasVideoOutput) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW + val template = if (outputs.videoOutput != null) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW val captureRequest = captureSession.device.createCaptureRequest(template) - outputs.forEach { output -> - if (output.outputType.isRepeating) { - Log.i(TAG, "Adding output surface ${output.outputType}..") - captureRequest.addTarget(output.surface) - } + outputs.previewOutput?.let { output -> + Log.i(TAG, "Adding output surface ${output.outputType}..") + captureRequest.addTarget(output.surface) + } + outputs.videoOutput?.let { output -> + Log.i(TAG, "Adding output surface ${output.outputType}..") + captureRequest.addTarget(output.surface) } - if (fps != null) { captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) @@ -407,23 +303,16 @@ class CameraSession(private val cameraManager: CameraManager, try { mutex.withLock { - val videoOutput = videoOutput - val photoOutput = photoOutput - val previewOutput = previewOutput val fps = fps val videoStabilizationMode = videoStabilizationMode val lowLightBoost = lowLightBoost val hdr = hdr + val outputs = outputs - if (previewOutput == null) { - cameraDevice?.tryClose() - return@withLock + if (outputs?.preview == null) { + throw Error("CameraSession doesn't have a Preview!") } - // 1. Create outputs for device (PREVIEW, PHOTO, VIDEO) - val outputs = getOutputs(cameraId, videoOutput, photoOutput, previewOutput) - if (outputs.isEmpty()) return - // 2. Open Camera Device val camera = getCameraDevice(cameraId) { reason -> isRunning = false @@ -459,8 +348,17 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Stopping Camera Session...") try { mutex.withLock { - val camera = cameraDevice ?: return - camera.close() + captureSession?.stopRepeating() + captureSession?.close() + captureSession = null + + outputs?.close() + outputs = null + + cameraDevice?.close() + cameraDevice = null + + isRunning = false Log.i(TAG, "Camera Session stopped!") } } catch (e: IllegalStateException) { diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 078dae3cd6..8d01f75c6e 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -18,6 +18,7 @@ import com.mrousavy.camera.extensions.displayRotation import com.mrousavy.camera.extensions.installHierarchyFitter import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.utils.CameraOutputs import kotlin.math.max import kotlin.math.min @@ -222,19 +223,20 @@ class CameraView(context: Context) : FrameLayout(context) { val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null - val previewSurface = previewSurface - - cameraSession.setOutputs( - // Photo Output - CameraSession.PhotoOutput(photo == true, targetPhotoSize), - // Video Output - CameraSession.VideoOutput(video == true, { image -> + val previewSurface = previewSurface ?: return + + val previewOutput = CameraOutputs.PreviewOutput(previewSurface) + val photoOutput = if (photo == true) { + CameraOutputs.PhotoOutput(targetPhotoSize) + } else null + val videoOutput = if (video == true) { + CameraOutputs.VideoOutput({ image -> val frame = Frame(image, System.currentTimeMillis(), inputRotation, false) onFrame(frame) - }, targetVideoSize), - // Preview Output - if (previewSurface?.isValid == true) CameraSession.PreviewOutput(true, previewSurface) else null - ) + }, targetVideoSize) + } else null + + cameraSession.setOutputs(previewOutput, photoOutput, videoOutput) } private fun configureFormat() { diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 0f360a9d54..db5d514934 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -1,8 +1,6 @@ package com.mrousavy.camera -import com.mrousavy.camera.extensions.ImageReaderOutput -import com.mrousavy.camera.extensions.SurfaceOutput -import com.mrousavy.camera.extensions.outputsToString +import com.mrousavy.camera.utils.CameraOutputs abstract class CameraError( /** @@ -54,7 +52,7 @@ class LowLightBoostNotContainedInFormatError : CameraError( class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") class CameraCannotBeOpenedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") -class CameraSessionCannotBeConfiguredError(cameraId: String, outputs: List) : CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera $cameraId! Outputs: ${outputsToString(outputs)}") +class CameraSessionCannotBeConfiguredError(cameraId: String, outputs: CameraOutputs) : CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera $cameraId! Outputs: $outputs") class CameraDisconnectedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error") class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index 4a4543546c..d53b74b788 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -4,16 +4,20 @@ import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager +import android.hardware.camera2.CameraMetadata import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration import android.media.ImageReader import android.os.Build import android.util.Log import android.view.Surface +import androidx.annotation.RequiresApi import com.mrousavy.camera.CameraQueues import com.mrousavy.camera.CameraSessionCannotBeConfiguredError import com.mrousavy.camera.parsers.parseHardwareLevel +import com.mrousavy.camera.utils.CameraOutputs import kotlinx.coroutines.suspendCancellableCoroutine +import java.io.Closeable import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException @@ -21,11 +25,11 @@ enum class SessionType { REGULAR, HIGH_SPEED; + @RequiresApi(Build.VERSION_CODES.P) fun toSessionType(): Int { - // TODO: Use actual enum when we are on API Level 28 return when(this) { - REGULAR -> 0 /* CameraDevice.SESSION_OPERATION_MODE_NORMAL */ - HIGH_SPEED -> 1 /* CameraDevice.SESSION_OPERATION_MODE_CONSTRAINED_HIGH_SPEED */ + REGULAR -> SessionConfiguration.SESSION_REGULAR + HIGH_SPEED -> SessionConfiguration.SESSION_HIGH_SPEED } } } @@ -36,31 +40,42 @@ enum class OutputType { PREVIEW, VIDEO_AND_PREVIEW; - fun toOutputType(): Long { - // TODO: Use actual enum when we are on API Level 28 + @RequiresApi(Build.VERSION_CODES.TIRAMISU) + fun toOutputType(): Int { return when(this) { - PHOTO -> 0x2 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_STILL_CAPTURE */ - VIDEO -> 0x3 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_VIDEO_RECORD */ - PREVIEW -> 0x1 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW */ - VIDEO_AND_PREVIEW -> 0x4 /* CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW_VIDEO_STILL */ + PHOTO -> CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_STILL_CAPTURE + VIDEO -> CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_VIDEO_RECORD + PREVIEW -> CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW + VIDEO_AND_PREVIEW -> CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_PREVIEW_VIDEO_STILL } } - - val isRepeating: Boolean - get() = arrayOf(VIDEO, PREVIEW, VIDEO_AND_PREVIEW).contains(this) } open class SurfaceOutput(val surface: Surface, val outputType: OutputType, - val dynamicRangeProfile: Long? = null) + private val dynamicRangeProfile: Long? = null) { + @RequiresApi(Build.VERSION_CODES.N) + fun toOutputConfiguration(characteristics: CameraCharacteristics): OutputConfiguration { + val result = OutputConfiguration(surface) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + if (dynamicRangeProfile != null) { + result.dynamicRangeProfile = dynamicRangeProfile + Log.i(TAG, "Using dynamic range profile ${result.dynamicRangeProfile} for $outputType output.") + } + if (supportsOutputType(characteristics, outputType)) { + result.streamUseCase = outputType.toOutputType().toLong() + Log.i(TAG, "Using optimized stream use case ${result.streamUseCase} for $outputType output.") + } + } + return result + } +} class ImageReaderOutput(val imageReader: ImageReader, outputType: OutputType, - dynamicRangeProfile: Long? = null): SurfaceOutput(imageReader.surface, outputType, dynamicRangeProfile) - -fun outputsToString(outputs: List): String { - return outputs.joinToString(", ", "[", "]") { output -> - if (output is ImageReaderOutput) "${output.outputType} (${output.imageReader.width} x ${output.imageReader.height} in format #${output.imageReader.imageFormat})" - else "${output.outputType}" + dynamicRangeProfile: Long? = null): Closeable, SurfaceOutput(imageReader.surface, outputType, dynamicRangeProfile) { + override fun close() { + Log.i(TAG, "Closing ${imageReader.width}x${imageReader.height} $outputType ImageReader..") + imageReader.close() } } @@ -68,7 +83,7 @@ private fun supportsOutputType(characteristics: CameraCharacteristics, outputTyp if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { val availableUseCases = characteristics.get(CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES) if (availableUseCases != null) { - if (availableUseCases.contains(outputType.toOutputType())) { + if (availableUseCases.contains(outputType.toOutputType().toLong())) { return true } } @@ -82,7 +97,7 @@ private var sessionId = 1000 suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, sessionType: SessionType, - outputs: List, + outputs: CameraOutputs, onClosed: (session: CameraCaptureSession) -> Unit, queue: CameraQueues.CameraQueue): CameraCaptureSession { return suspendCancellableCoroutine { continuation -> @@ -90,7 +105,7 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! val sessionId = sessionId++ Log.i(TAG, "Camera $id: Creating Capture Session #$sessionId... " + - "Hardware Level: ${parseHardwareLevel(hardwareLevel)} | Outputs: ${outputsToString(outputs)}") + "Hardware Level: ${parseHardwareLevel(hardwareLevel)} | Outputs: $outputs") val callback = object: CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) { @@ -113,24 +128,14 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { // API >= 24 val outputConfigurations = arrayListOf() - for (output in outputs) { - if (!output.surface.isValid) { - Log.w(TAG, "Tried to add ${output.outputType} output, but Surface was invalid! Skipping this output..") - continue - } - val result = OutputConfiguration(output.surface) - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - if (output.dynamicRangeProfile != null) { - result.dynamicRangeProfile = output.dynamicRangeProfile - Log.i(TAG, "Using dynamic range profile ${result.dynamicRangeProfile} for ${output.outputType} output.") - } - if (supportsOutputType(characteristics, output.outputType)) { - result.streamUseCase = output.outputType.toOutputType() - Log.i(TAG, "Using optimized stream use case ${result.streamUseCase} for ${output.outputType} output.") - } - } - outputConfigurations.add(result) + outputs.previewOutput?.let { output -> + outputConfigurations.add(output.toOutputConfiguration(characteristics)) + } + outputs.photoOutput?.let { output -> + outputConfigurations.add(output.toOutputConfiguration(characteristics)) + } + outputs.videoOutput?.let { output -> + outputConfigurations.add(output.toOutputConfiguration(characteristics)) } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { @@ -146,7 +151,11 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, } else { // API <24 Log.i(TAG, "Using legacy API (<24)") - this.createCaptureSession(outputs.map { it.surface }, callback, queue.handler) + val surfaces = arrayListOf() + outputs.previewOutput?.let { surfaces.add(it.surface) } + outputs.photoOutput?.let { surfaces.add(it.surface) } + outputs.videoOutput?.let { surfaces.add(it.surface) } + this.createCaptureSession(surfaces, callback, queue.handler) } } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt new file mode 100644 index 0000000000..2ded6847ec --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt @@ -0,0 +1,136 @@ +package com.mrousavy.camera.utils + +import android.graphics.ImageFormat +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraManager +import android.media.Image +import android.media.ImageReader +import android.util.Log +import android.util.Size +import android.view.Surface +import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.extensions.ImageReaderOutput +import com.mrousavy.camera.extensions.OutputType +import com.mrousavy.camera.extensions.SurfaceOutput +import com.mrousavy.camera.extensions.closestToOrMax +import java.io.Closeable + +class CameraOutputs(val cameraId: String, + cameraManager: CameraManager, + val preview: PreviewOutput? = null, + val photo: PhotoOutput? = null, + val video: VideoOutput? = null, + val callback: Callback): Closeable { + companion object { + private const val TAG = "CameraOutputs" + private const val VIDEO_OUTPUT_BUFFER_SIZE = 3 + private const val PHOTO_OUTPUT_BUFFER_SIZE = 3 + } + + data class PreviewOutput(val surface: Surface) + data class PhotoOutput(val targetSize: Size? = null, + val format: Int = ImageFormat.JPEG) + data class VideoOutput(val onFrame: (image: Image) -> Unit, + val targetSize: Size? = null, + val format: Int = ImageFormat.YUV_420_888) + + interface Callback { + fun onPhotoCaptured(image: Image) + } + + var previewOutput: SurfaceOutput? = null + private set + var photoOutput: ImageReaderOutput? = null + private set + var videoOutput: ImageReaderOutput? = null + private set + + val size: Int + get() { + var size = 0 + if (previewOutput != null) size++ + if (photoOutput != null) size++ + if (videoOutput != null) size++ + return size + } + + override fun equals(other: Any?): Boolean { + if (other !is CameraOutputs) return false + return this.cameraId == other.cameraId + && (this.preview == null) == (other.preview == null) + && this.photo?.targetSize == other.photo?.targetSize + && this.photo?.format == other.photo?.format + && this.video?.targetSize == other.video?.targetSize + && this.video?.format == other.video?.format + } + + override fun hashCode(): Int { + var result = cameraId.hashCode() + result += (preview?.hashCode() ?: 0) + result += (photo?.hashCode() ?: 0) + result += (video?.hashCode() ?: 0) + return result + } + + override fun close() { + photoOutput?.close() + videoOutput?.close() + } + + override fun toString(): String { + val strings = arrayListOf() + photoOutput?.let { + strings.add("${it.outputType} (${it.imageReader.width} x ${it.imageReader.height} in format #${it.imageReader.imageFormat})") + } + videoOutput?.let { + strings.add("${it.outputType} (${it.imageReader.width} x ${it.imageReader.height} in format #${it.imageReader.imageFormat})") + } + previewOutput?.let { + strings.add("${it.outputType}") + } + return strings.joinToString(", ", "[", "]") + } + + init { + val characteristics = cameraManager.getCameraCharacteristics(cameraId) + val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + + Log.i(TAG, "Preparing Outputs for Camera $cameraId...") + + // Preview output: Low resolution repeating images (SurfaceView) + if (preview != null) { + Log.i(TAG, "Adding native preview view output.") + previewOutput = SurfaceOutput(preview.surface, OutputType.PREVIEW) + } + + // Photo output: High quality still images (takePhoto()) + if (photo != null) { + val size = config.getOutputSizes(photo.format).closestToOrMax(photo.targetSize) + + val imageReader = ImageReader.newInstance(size.width, size.height, photo.format, PHOTO_OUTPUT_BUFFER_SIZE) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireLatestImage() ?: return@setOnImageAvailableListener + callback.onPhotoCaptured(image) + }, CameraQueues.cameraQueue.handler) + + Log.i(TAG, "Adding ${size.width}x${size.height} photo output. (Format: $photo.format)") + photoOutput = ImageReaderOutput(imageReader, OutputType.PHOTO) + } + + // Video output: High resolution repeating images (startRecording() or useFrameProcessor()) + if (video != null) { + val size = config.getOutputSizes(video.format).closestToOrMax(video.targetSize) + + val imageReader = ImageReader.newInstance(size.width, size.height, video.format, VIDEO_OUTPUT_BUFFER_SIZE) + imageReader.setOnImageAvailableListener({ reader -> + val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener + video.onFrame(image) + }, CameraQueues.videoQueue.handler) + + Log.i(TAG, "Adding ${size.width}x${size.height} video output. (Format: $video.format)") + videoOutput = ImageReaderOutput(imageReader, OutputType.VIDEO) + } + + Log.i(TAG, "Prepared $size Outputs for Camera $cameraId!") + } +} From 9829dc4b1553e452bf51206f94af680ed89efdee Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 5 Aug 2023 12:57:49 +0200 Subject: [PATCH 070/180] fix: Keep Outputs --- .../java/com/mrousavy/camera/CameraSession.kt | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 65da39b2c7..72f9d6c050 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -295,6 +295,18 @@ class CameraSession(private val cameraManager: CameraManager, return captureRequest.build() } + private fun destroy() { + Log.i(TAG, "Destroying session..") + captureSession?.stopRepeating() + captureSession?.close() + captureSession = null + + cameraDevice?.close() + cameraDevice = null + + isRunning = false + } + private suspend fun startRunning() { isRunning = false val cameraId = cameraId ?: return @@ -310,7 +322,9 @@ class CameraSession(private val cameraManager: CameraManager, val outputs = outputs if (outputs?.preview == null) { - throw Error("CameraSession doesn't have a Preview!") + Log.i(TAG, "CameraSession doesn't have a Preview, canceling..") + destroy() + return@withLock } // 2. Open Camera Device @@ -348,17 +362,7 @@ class CameraSession(private val cameraManager: CameraManager, Log.i(TAG, "Stopping Camera Session...") try { mutex.withLock { - captureSession?.stopRepeating() - captureSession?.close() - captureSession = null - - outputs?.close() - outputs = null - - cameraDevice?.close() - cameraDevice = null - - isRunning = false + destroy() Log.i(TAG, "Camera Session stopped!") } } catch (e: IllegalStateException) { From 21bb04ce642394c98112cc689e7cb30827c24722 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 5 Aug 2023 13:01:45 +0200 Subject: [PATCH 071/180] Refactor into single method --- .../java/com/mrousavy/camera/CameraSession.kt | 23 ++++++++----------- .../java/com/mrousavy/camera/CameraView.kt | 16 ++++--------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 72f9d6c050..236a23ad44 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -96,26 +96,23 @@ class CameraSession(private val cameraManager: CameraManager, } } - /** - * Configure the outputs of the Camera. - */ - fun setOutputs(preview: CameraOutputs.PreviewOutput? = null, - photo: CameraOutputs.PhotoOutput? = null, - video: CameraOutputs.VideoOutput? = null) { - Log.i(TAG, "Setting Outputs...") - val cameraId = cameraId ?: throw NoCameraDeviceError() - val newOutputs = CameraOutputs(cameraId, + fun configureSession(cameraId: String, + preview: CameraOutputs.PreviewOutput? = null, + photo: CameraOutputs.PhotoOutput? = null, + video: CameraOutputs.VideoOutput? = null) { + Log.i(TAG, "Configuring Session for Camera $cameraId...") + val outputs = CameraOutputs(cameraId, cameraManager, preview, photo, video, this) - if (this.outputs == newOutputs) { - Log.i(TAG, "Outputs didn't change, skipping restart..") - return + if (this.cameraId == cameraId && this.outputs == outputs && isActive == isRunning) { + Log.i(TAG, "Nothing changed in configuration, canceling..") } - this.outputs = newOutputs + this.cameraId = cameraId + this.outputs = outputs launch { startRunning() } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 8d01f75c6e..308139487c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -148,11 +148,10 @@ class CameraView(context: Context) : FrameLayout(context) { } private fun setupPreviewView() { - val cameraId = cameraId ?: return - if (previewType == "native") { removeView(this.previewView) + val cameraId = cameraId ?: throw NoCameraDeviceError() val previewView = NativePreviewView(cameraManager, cameraId, context) { surface -> previewSurface = surface configureSession() @@ -169,7 +168,6 @@ class CameraView(context: Context) : FrameLayout(context) { Log.i(TAG, "Props changed: $changedProps") try { val shouldReconfigurePreview = changedProps.containsAny(propsThatRequirePreviewReconfiguration) - val shouldReconfigureDevice = changedProps.contains("cameraId") val shouldReconfigureSession = shouldReconfigurePreview || changedProps.containsAny(propsThatRequireSessionReconfiguration) val shouldReconfigureFormat = shouldReconfigureSession || changedProps.containsAny(propsThatRequireFormatReconfiguration) val shouldReconfigureZoom = /* TODO: When should we reconfigure this? */ shouldReconfigureSession || changedProps.contains("zoom") @@ -180,9 +178,6 @@ class CameraView(context: Context) : FrameLayout(context) { if (shouldReconfigurePreview) { setupPreviewView() } - if (shouldReconfigureDevice) { - configureDevice() - } if (shouldReconfigureSession) { configureSession() } @@ -209,17 +204,14 @@ class CameraView(context: Context) : FrameLayout(context) { } } - private fun configureDevice() { + private fun configureSession() { Log.i(TAG, "Configuring Camera Device...") + if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { throw CameraPermissionError() } val cameraId = cameraId ?: throw NoCameraDeviceError() - cameraSession.setInputDevice(cameraId) - } - - private fun configureSession() { val format = format val targetVideoSize = if (format != null) Size(format.getInt("videoWidth"), format.getInt("videoHeight")) else null val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null @@ -236,7 +228,7 @@ class CameraView(context: Context) : FrameLayout(context) { }, targetVideoSize) } else null - cameraSession.setOutputs(previewOutput, photoOutput, videoOutput) + cameraSession.configureSession(cameraId, previewOutput, photoOutput, videoOutput) } private fun configureFormat() { From c89d21fa441b89d66feebc446edf4c1cd77961f6 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 5 Aug 2023 13:03:41 +0200 Subject: [PATCH 072/180] Update CameraView.kt --- .../java/com/mrousavy/camera/CameraView.kt | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 308139487c..03710e2762 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -76,7 +76,7 @@ class CameraView(context: Context) : FrameLayout(context) { var videoStabilizationMode: String? = null var hdr: Boolean? = null // nullable bool var lowLightBoost: Boolean? = null // nullable bool - var previewType: String = "native" + var previewType: String = "none" // other props var isActive = false var torch = "off" @@ -148,19 +148,26 @@ class CameraView(context: Context) : FrameLayout(context) { } private fun setupPreviewView() { - if (previewType == "native") { - removeView(this.previewView) + when (previewType) { + "none" -> { + removeView(this.previewView) + this.previewView = null + } + "native" -> { + removeView(this.previewView) - val cameraId = cameraId ?: throw NoCameraDeviceError() - val previewView = NativePreviewView(cameraManager, cameraId, context) { surface -> - previewSurface = surface - configureSession() + val cameraId = cameraId ?: throw NoCameraDeviceError() + val previewView = NativePreviewView(cameraManager, cameraId, context) { surface -> + previewSurface = surface + configureSession() + } + previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) + addView(previewView) + this.previewView = previewView + } + "skia" -> { + throw Error("Skia is not yet implemented on Android!") } - previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) - addView(previewView) - this.previewView = previewView - } else { - throw Error("Skia is not yet implemented on Android!") } } From 42ec0ad2238fe5b65a6e07b93515c6a4719e1f48 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 5 Aug 2023 14:08:55 +0200 Subject: [PATCH 073/180] Use Enums for type safety --- .../java/com/mrousavy/camera/CameraSession.kt | 22 ++--- .../mrousavy/camera/CameraView+TakePhoto.kt | 22 ++--- .../java/com/mrousavy/camera/CameraView.kt | 40 +++----- .../com/mrousavy/camera/CameraViewManager.kt | 30 +++--- .../com/mrousavy/camera/CameraViewModule.kt | 17 ++-- .../main/java/com/mrousavy/camera/Errors.kt | 5 +- .../CameraDevice+createCaptureSession.kt | 3 +- .../CameraDevice+createPhotoCaptureRequest.kt | 14 ++- .../extensions/CameraManager+openCamera.kt | 12 +-- .../mrousavy/camera/frameprocessor/Frame.java | 23 ++--- .../camera/parsers/CameraDeviceError.kt | 25 +++++ .../camera/parsers/CameraError+String.kt | 14 --- .../java/com/mrousavy/camera/parsers/Flash.kt | 18 ++++ .../com/mrousavy/camera/parsers/Format.kt | 23 +++++ .../camera/parsers/HardwareLevel+String.kt | 14 --- .../mrousavy/camera/parsers/HardwareLevel.kt | 24 +++++ .../camera/parsers/ImageFormat+String.kt | 51 ---------- .../mrousavy/camera/parsers/JSUnionValue.kt | 9 ++ .../com/mrousavy/camera/parsers/LensFacing.kt | 20 ++++ .../camera/parsers/LenseFacing+String.kt | 15 --- .../mrousavy/camera/parsers/Orientation.kt | 40 ++++++++ .../camera/parsers/PermissionStatus+String.kt | 11 --- .../camera/parsers/PermissionStatus.kt | 19 ++++ .../mrousavy/camera/parsers/PreviewType.kt | 18 ++++ .../camera/parsers/QualityPrioritization.kt | 18 ++++ .../java/com/mrousavy/camera/parsers/Torch.kt | 16 ++++ .../parsers/VideoStabilizationMode+String.kt | 36 ------- .../camera/parsers/VideoStabilizationMode.kt | 59 ++++++++++++ .../camera/utils/CameraDeviceDetails.kt | 32 +++---- docs/docs/guides/MOCKING.mdx | 94 +++++++++---------- docs/docs/guides/SETUP.mdx | 4 +- example/src/App.tsx | 2 +- example/src/CameraPage.tsx | 2 +- example/src/PermissionsPage.tsx | 6 +- .../AVAuthorizationStatus+descriptor.swift | 2 +- src/Camera.tsx | 4 +- 36 files changed, 447 insertions(+), 317 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/CameraDeviceError.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/Flash.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/Format.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/JSUnionValue.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/LensFacing.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/parsers/LenseFacing+String.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/PreviewType.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/QualityPrioritization.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/Torch.kt delete mode 100644 android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode.kt diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 236a23ad44..4ccd4b1e6a 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -10,15 +10,16 @@ import android.media.Image import android.os.Build import android.util.Log import android.util.Range -import com.mrousavy.camera.extensions.FlashMode -import com.mrousavy.camera.extensions.QualityPrioritization import com.mrousavy.camera.extensions.SessionType import com.mrousavy.camera.extensions.capture import com.mrousavy.camera.extensions.createCaptureSession import com.mrousavy.camera.extensions.createPhotoCaptureRequest import com.mrousavy.camera.extensions.openCamera import com.mrousavy.camera.extensions.tryClose -import com.mrousavy.camera.parsers.getVideoStabilizationMode +import com.mrousavy.camera.parsers.CameraDeviceError +import com.mrousavy.camera.parsers.Flash +import com.mrousavy.camera.parsers.QualityPrioritization +import com.mrousavy.camera.parsers.VideoStabilizationMode import com.mrousavy.camera.utils.CameraOutputs import com.mrousavy.camera.utils.ExifUtils import com.mrousavy.camera.utils.PhotoOutputSynchronizer @@ -59,7 +60,7 @@ class CameraSession(private val cameraManager: CameraManager, // configureFormat(..) private var fps: Int? = null - private var videoStabilizationMode: String? = null + private var videoStabilizationMode: VideoStabilizationMode? = null private var lowLightBoost: Boolean? = null private var hdr: Boolean? = null @@ -122,7 +123,7 @@ class CameraSession(private val cameraManager: CameraManager, * Configures various format settings such as FPS, Video Stabilization, HDR or Night Mode. */ fun configureFormat(fps: Int? = null, - videoStabilizationMode: String? = null, + videoStabilizationMode: VideoStabilizationMode? = null, hdr: Boolean? = null, lowLightBoost: Boolean? = null) { Log.i(TAG, "Setting Format (fps: $fps | videoStabilization: $videoStabilizationMode | hdr: $hdr | lowLightBoost: $lowLightBoost)...") @@ -153,7 +154,7 @@ class CameraSession(private val cameraManager: CameraManager, } suspend fun takePhoto(qualityPrioritization: QualityPrioritization, - flashMode: FlashMode, + flashMode: Flash, enableRedEyeReduction: Boolean, enableAutoStabilization: Boolean): CapturedPhoto { val captureSession = captureSession ?: throw CameraNotReadyError() @@ -259,7 +260,7 @@ class CameraSession(private val cameraManager: CameraManager, private fun getPreviewCaptureRequest(captureSession: CameraCaptureSession, outputs: CameraOutputs, fps: Int? = null, - videoStabilizationMode: String? = null, + videoStabilizationMode: VideoStabilizationMode? = null, lowLightBoost: Boolean? = null, hdr: Boolean? = null): CaptureRequest { val template = if (outputs.videoOutput != null) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW @@ -277,9 +278,8 @@ class CameraSession(private val cameraManager: CameraManager, captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) } if (videoStabilizationMode != null) { - val stabilizationMode = getVideoStabilizationMode(videoStabilizationMode) - captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, stabilizationMode.digitalMode) - captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, stabilizationMode.opticalMode) + captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, videoStabilizationMode.toDigitalStabilizationMode()) + captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, videoStabilizationMode.toOpticalStabilizationMode()) } if (lowLightBoost == true) { captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) @@ -333,7 +333,7 @@ class CameraSession(private val cameraManager: CameraManager, // 3. Create capture session with outputs val session = getCaptureSession(camera, outputs) { isRunning = false - onError(CameraDisconnectedError(cameraId, "session-closed")) + onError(CameraDisconnectedError(cameraId, CameraDeviceError.DISCONNECTED)) } // 4. Create repeating request (configures FPS, HDR, etc.) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index 291964949b..2c8872fac7 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -8,8 +8,8 @@ import android.util.Log import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap -import com.mrousavy.camera.extensions.FlashMode -import com.mrousavy.camera.extensions.QualityPrioritization +import com.mrousavy.camera.parsers.Flash +import com.mrousavy.camera.parsers.QualityPrioritization import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import java.io.File @@ -22,24 +22,14 @@ suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = corouti val options = optionsMap.toHashMap() Log.i(TAG, "Taking photo... Options: $options") - val qualityPrioritization = options["qualityPrioritization"] as? String - val flash = options["flash"] as? String + val qualityPrioritization = options["qualityPrioritization"] as? String ?: "balanced" + val flash = options["flash"] as? String ?: "off" val enableAutoRedEyeReduction = options["enableAutoRedEyeReduction"] == true val enableAutoStabilization = options["enableAutoStabilization"] == true val skipMetadata = options["skipMetadata"] == true - val flashMode = when (flash) { - "off" -> FlashMode.OFF - "on" -> FlashMode.ON - "auto" -> FlashMode.AUTO - else -> FlashMode.AUTO - } - val qualityPrioritizationMode = when (qualityPrioritization) { - "speed" -> QualityPrioritization.SPEED - "balanced" -> QualityPrioritization.BALANCED - "quality" -> QualityPrioritization.QUALITY - else -> QualityPrioritization.BALANCED - } + val flashMode = Flash.fromUnionValue(flash) + val qualityPrioritizationMode = QualityPrioritization.fromUnionValue(qualityPrioritization) val photo = cameraSession.takePhoto(qualityPrioritizationMode, flashMode, diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 03710e2762..a3e0499ec1 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -18,6 +18,10 @@ import com.mrousavy.camera.extensions.displayRotation import com.mrousavy.camera.extensions.installHierarchyFitter import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.parsers.Orientation +import com.mrousavy.camera.parsers.PreviewType +import com.mrousavy.camera.parsers.Torch +import com.mrousavy.camera.parsers.VideoStabilizationMode import com.mrousavy.camera.utils.CameraOutputs import kotlin.math.max import kotlin.math.min @@ -73,15 +77,15 @@ class CameraView(context: Context) : FrameLayout(context) { // props that require format reconfiguring var format: ReadableMap? = null var fps: Int? = null - var videoStabilizationMode: String? = null + var videoStabilizationMode: VideoStabilizationMode? = null var hdr: Boolean? = null // nullable bool var lowLightBoost: Boolean? = null // nullable bool - var previewType: String = "none" + var previewType: PreviewType = PreviewType.NONE // other props var isActive = false - var torch = "off" + var torch: Torch = Torch.OFF var zoom: Float = 1f // in "factor" - var orientation: String? = null + var orientation: Orientation? = null // private properties private var isMounted = false @@ -95,25 +99,9 @@ class CameraView(context: Context) : FrameLayout(context) { var frameProcessor: FrameProcessor? = null private val inputRotation: Int - get() { - return context.displayRotation - } + get() = context.displayRotation private val outputRotation: Int - get() { - if (orientation != null) { - // user is overriding output orientation - return when (orientation!!) { - "portrait" -> Surface.ROTATION_0 - "landscapeRight" -> Surface.ROTATION_90 - "portraitUpsideDown" -> Surface.ROTATION_180 - "landscapeLeft" -> Surface.ROTATION_270 - else -> throw InvalidTypeScriptUnionError("orientation", orientation!!) - } - } else { - // use same as input rotation - return inputRotation - } - } + get() = orientation?.toSurfaceRotation() ?: inputRotation private var minZoom: Float = 1f private var maxZoom: Float = 1f @@ -149,11 +137,11 @@ class CameraView(context: Context) : FrameLayout(context) { private fun setupPreviewView() { when (previewType) { - "none" -> { + PreviewType.NONE -> { removeView(this.previewView) this.previewView = null } - "native" -> { + PreviewType.NATIVE -> { removeView(this.previewView) val cameraId = cameraId ?: throw NoCameraDeviceError() @@ -165,7 +153,7 @@ class CameraView(context: Context) : FrameLayout(context) { addView(previewView) this.previewView = previewView } - "skia" -> { + PreviewType.SKIA -> { throw Error("Skia is not yet implemented on Android!") } } @@ -230,7 +218,7 @@ class CameraView(context: Context) : FrameLayout(context) { } else null val videoOutput = if (video == true) { CameraOutputs.VideoOutput({ image -> - val frame = Frame(image, System.currentTimeMillis(), inputRotation, false) + val frame = Frame(image, System.currentTimeMillis(), Orientation.fromRotationDegrees(inputRotation), false) onFrame(frame) }, targetVideoSize) } else null diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt index 54caa80ea7..18a2becc41 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt @@ -3,9 +3,13 @@ package com.mrousavy.camera import com.facebook.react.bridge.ReactApplicationContext import com.facebook.react.bridge.ReadableMap import com.facebook.react.common.MapBuilder -import com.facebook.react.uimanager.ViewGroupManager import com.facebook.react.uimanager.ThemedReactContext +import com.facebook.react.uimanager.ViewGroupManager import com.facebook.react.uimanager.annotations.ReactProp +import com.mrousavy.camera.parsers.Orientation +import com.mrousavy.camera.parsers.PreviewType +import com.mrousavy.camera.parsers.Torch +import com.mrousavy.camera.parsers.VideoStabilizationMode @Suppress("unused") class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManager() { @@ -77,16 +81,18 @@ class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManage @ReactProp(name = "videoStabilizationMode") fun setVideoStabilizationMode(view: CameraView, videoStabilizationMode: String?) { - if (view.videoStabilizationMode != videoStabilizationMode) + val newMode = VideoStabilizationMode.fromUnionValue(videoStabilizationMode) + if (view.videoStabilizationMode != newMode) addChangedPropToTransaction(view, "videoStabilizationMode") - view.videoStabilizationMode = videoStabilizationMode + view.videoStabilizationMode = newMode } @ReactProp(name = "previewType") - fun setPreviewType(view: CameraView, previewType: String?) { - if (view.previewType != previewType) + fun setPreviewType(view: CameraView, previewType: String) { + val newMode = PreviewType.fromUnionValue(previewType) + if (view.previewType != newMode) addChangedPropToTransaction(view, "previewType") - view.previewType = previewType ?: "native" + view.previewType = newMode } @ReactProp(name = "enableHighQualityPhotos") @@ -143,9 +149,10 @@ class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManage @ReactProp(name = "torch") fun setTorch(view: CameraView, torch: String) { - if (view.torch != torch) + val newMode = Torch.fromUnionValue(torch) + if (view.torch != newMode) addChangedPropToTransaction(view, "torch") - view.torch = torch + view.torch = newMode } @ReactProp(name = "zoom") @@ -157,10 +164,11 @@ class CameraViewManager(reactContext: ReactApplicationContext) : ViewGroupManage } @ReactProp(name = "orientation") - fun setOrientation(view: CameraView, orientation: String) { - if (view.orientation != orientation) + fun setOrientation(view: CameraView, orientation: String?) { + val newMode = Orientation.fromUnionValue(orientation) + if (view.orientation != newMode) addChangedPropToTransaction(view, "orientation") - view.orientation = orientation + view.orientation = newMode } companion object { diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index 59030c9d58..37ec5cd275 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -166,21 +166,24 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod fun getCameraPermissionStatus(promise: Promise) { + val status = ContextCompat.checkSelfPermission(reactApplicationContext, Manifest.permission.CAMERA) - promise.resolve(parsePermissionStatus(status)) + val parsed = PermissionStatus.fromPermissionStatus(status) + promise.resolve(parsed.unionValue) } @ReactMethod fun getMicrophonePermissionStatus(promise: Promise) { val status = ContextCompat.checkSelfPermission(reactApplicationContext, Manifest.permission.RECORD_AUDIO) - promise.resolve(parsePermissionStatus(status)) + val parsed = PermissionStatus.fromPermissionStatus(status) + promise.resolve(parsed.unionValue) } @ReactMethod fun requestCameraPermission(promise: Promise) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { // API 21 and below always grants permission on app install - return promise.resolve("authorized") + return promise.resolve(PermissionStatus.GRANTED.unionValue) } val activity = reactApplicationContext.currentActivity @@ -189,7 +192,8 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ val listener = PermissionListener { requestCode: Int, _: Array, grantResults: IntArray -> if (requestCode == currentRequestCode) { val permissionStatus = if (grantResults.isNotEmpty()) grantResults[0] else PackageManager.PERMISSION_DENIED - promise.resolve(parsePermissionStatus(permissionStatus)) + val parsed = PermissionStatus.fromPermissionStatus(permissionStatus) + promise.resolve(parsed.unionValue) return@PermissionListener true } return@PermissionListener false @@ -204,7 +208,7 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ fun requestMicrophonePermission(promise: Promise) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { // API 21 and below always grants permission on app install - return promise.resolve("authorized") + return promise.resolve(PermissionStatus.GRANTED.unionValue) } val activity = reactApplicationContext.currentActivity @@ -213,7 +217,8 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ val listener = PermissionListener { requestCode: Int, _: Array, grantResults: IntArray -> if (requestCode == currentRequestCode) { val permissionStatus = if (grantResults.isNotEmpty()) grantResults[0] else PackageManager.PERMISSION_DENIED - promise.resolve(parsePermissionStatus(permissionStatus)) + val parsed = PermissionStatus.fromPermissionStatus(permissionStatus) + promise.resolve(parsed.unionValue) return@PermissionListener true } return@PermissionListener false diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index db5d514934..66b49e9a00 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -1,5 +1,6 @@ package com.mrousavy.camera +import com.mrousavy.camera.parsers.CameraDeviceError import com.mrousavy.camera.utils.CameraOutputs abstract class CameraError( @@ -51,9 +52,9 @@ class LowLightBoostNotContainedInFormatError : CameraError( ) class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") -class CameraCannotBeOpenedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") +class CameraCannotBeOpenedError(cameraId: String, error: CameraDeviceError) : CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error") class CameraSessionCannotBeConfiguredError(cameraId: String, outputs: CameraOutputs) : CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera $cameraId! Outputs: $outputs") -class CameraDisconnectedError(cameraId: String, error: String? = "(none)") : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error") +class CameraDisconnectedError(cameraId: String, error: CameraDeviceError) : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error") class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.") diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index d53b74b788..5cab786f01 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -14,7 +14,6 @@ import android.view.Surface import androidx.annotation.RequiresApi import com.mrousavy.camera.CameraQueues import com.mrousavy.camera.CameraSessionCannotBeConfiguredError -import com.mrousavy.camera.parsers.parseHardwareLevel import com.mrousavy.camera.utils.CameraOutputs import kotlinx.coroutines.suspendCancellableCoroutine import java.io.Closeable @@ -105,7 +104,7 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)!! val sessionId = sessionId++ Log.i(TAG, "Camera $id: Creating Capture Session #$sessionId... " + - "Hardware Level: ${parseHardwareLevel(hardwareLevel)} | Outputs: $outputs") + "Hardware Level: $hardwareLevel} | Outputs: $outputs") val callback = object: CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) { diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt index c38ea0bb49..6e888f0bd4 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt @@ -5,15 +5,13 @@ import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.view.Surface - -enum class FlashMode { OFF, ON, AUTO } - -enum class QualityPrioritization { SPEED, BALANCED, QUALITY } +import com.mrousavy.camera.parsers.Flash +import com.mrousavy.camera.parsers.QualityPrioritization fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, surface: Surface, qualityPrioritization: QualityPrioritization, - flashMode: FlashMode, + flashMode: Flash, enableRedEyeReduction: Boolean, enableAutoStabilization: Boolean): CaptureRequest { val captureRequest = when (qualityPrioritization) { @@ -35,15 +33,15 @@ fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, when (flashMode) { // Set the Flash Mode - FlashMode.OFF -> { + Flash.OFF -> { captureRequest[CaptureRequest.FLASH_MODE] = CaptureRequest.FLASH_MODE_OFF captureRequest[CaptureRequest.CONTROL_AE_MODE] = CaptureRequest.CONTROL_AE_MODE_ON } - FlashMode.ON -> { + Flash.ON -> { captureRequest[CaptureRequest.FLASH_MODE] = CaptureRequest.FLASH_MODE_SINGLE captureRequest[CaptureRequest.CONTROL_AE_MODE] = CaptureRequest.CONTROL_AE_MODE_ON_ALWAYS_FLASH } - FlashMode.AUTO -> { + Flash.AUTO -> { if (enableRedEyeReduction) { captureRequest[CaptureRequest.CONTROL_AE_MODE] = CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH_REDEYE } else { diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt index d838ace777..4975cf3b04 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraManager+openCamera.kt @@ -8,7 +8,7 @@ import android.util.Log import com.mrousavy.camera.CameraCannotBeOpenedError import com.mrousavy.camera.CameraDisconnectedError import com.mrousavy.camera.CameraQueues -import com.mrousavy.camera.parsers.parseCameraError +import com.mrousavy.camera.parsers.CameraDeviceError import kotlinx.coroutines.suspendCancellableCoroutine import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException @@ -30,21 +30,21 @@ suspend fun CameraManager.openCamera(cameraId: String, override fun onDisconnected(camera: CameraDevice) { Log.i(TAG, "Camera $cameraId: Disconnected!") - val errorCode = "disconnected" if (continuation.isActive) { - continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, errorCode)) + continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, CameraDeviceError.DISCONNECTED)) } else { - onDisconnected(camera, CameraDisconnectedError(cameraId, errorCode)) + onDisconnected(camera, CameraDisconnectedError(cameraId, CameraDeviceError.DISCONNECTED)) } camera.tryClose() } override fun onError(camera: CameraDevice, errorCode: Int) { Log.e(TAG, "Camera $cameraId: Error! $errorCode") + val error = CameraDeviceError.fromCameraDeviceError(errorCode) if (continuation.isActive) { - continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, parseCameraError(errorCode))) + continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, error)) } else { - onDisconnected(camera, CameraDisconnectedError(cameraId, parseCameraError(errorCode))) + onDisconnected(camera, CameraDisconnectedError(cameraId, error)) } camera.tryClose() } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index e8eb945917..233eec1969 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -1,20 +1,20 @@ package com.mrousavy.camera.frameprocessor; -import static com.mrousavy.camera.parsers.ImageFormat_StringKt.parseImageFormat; - import android.graphics.ImageFormat; -import android.graphics.PixelFormat; import android.media.Image; import com.facebook.proguard.annotations.DoNotStrip; +import com.mrousavy.camera.parsers.Format; +import com.mrousavy.camera.parsers.Orientation; + import java.nio.ByteBuffer; public class Frame { private final Image image; private final boolean isMirrored; private final long timestamp; - private final int orientation; + private final Orientation orientation; - public Frame(Image image, long timestamp, int orientation, boolean isMirrored) { + public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) { this.image = image; this.timestamp = timestamp; this.orientation = orientation; @@ -66,21 +66,14 @@ public long getTimestamp() { @SuppressWarnings("unused") @DoNotStrip public String getOrientation() { - // TODO: Check if this works as expected - int rotation = orientation; - if (rotation >= 45 && rotation < 135) - return "landscapeRight"; - if (rotation >= 135 && rotation < 225) - return "portraitUpsideDown"; - if (rotation >= 225 && rotation < 315) - return "landscapeLeft"; - return "portrait"; + return orientation.getUnionValue(); } @SuppressWarnings("unused") @DoNotStrip public String getPixelFormat() { - return parseImageFormat(image.getFormat()); + Format format = Format.Companion.fromImageFormat(image.getFormat()); + return format.getUnionValue(); } @SuppressWarnings("unused") diff --git a/android/src/main/java/com/mrousavy/camera/parsers/CameraDeviceError.kt b/android/src/main/java/com/mrousavy/camera/parsers/CameraDeviceError.kt new file mode 100644 index 0000000000..a2f3c448c7 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/CameraDeviceError.kt @@ -0,0 +1,25 @@ +package com.mrousavy.camera.parsers + +import android.hardware.camera2.CameraDevice + +enum class CameraDeviceError(override val unionValue: String): JSUnionValue { + CAMERA_ALREADY_IN_USE("camera-already-in-use"), + TOO_MANY_OPEN_CAMERAS("too-many-open-cameras"), + CAMERA_IS_DISABLED_BY_ANDROID("camera-is-disabled-by-android"), + UNKNOWN_CAMERA_DEVICE_ERROR("unknown-camera-device-error"), + UNKNOWN_FATAL_CAMERA_SERVICE_ERROR("unknown-fatal-camera-service-error"), + DISCONNECTED("camera-has-been-disconnected"); + + companion object { + fun fromCameraDeviceError(cameraDeviceError: Int): CameraDeviceError { + return when (cameraDeviceError) { + CameraDevice.StateCallback.ERROR_CAMERA_IN_USE -> CAMERA_ALREADY_IN_USE + CameraDevice.StateCallback.ERROR_MAX_CAMERAS_IN_USE -> TOO_MANY_OPEN_CAMERAS + CameraDevice.StateCallback.ERROR_CAMERA_DISABLED -> CAMERA_IS_DISABLED_BY_ANDROID + CameraDevice.StateCallback.ERROR_CAMERA_DEVICE -> UNKNOWN_CAMERA_DEVICE_ERROR + CameraDevice.StateCallback.ERROR_CAMERA_SERVICE -> UNKNOWN_FATAL_CAMERA_SERVICE_ERROR + else -> UNKNOWN_CAMERA_DEVICE_ERROR + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt deleted file mode 100644 index 3388206133..0000000000 --- a/android/src/main/java/com/mrousavy/camera/parsers/CameraError+String.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.mrousavy.camera.parsers - -import android.hardware.camera2.CameraDevice - -fun parseCameraError(error: Int): String { - return when (error) { - CameraDevice.StateCallback.ERROR_CAMERA_IN_USE -> "camera-already-in-use" - CameraDevice.StateCallback.ERROR_MAX_CAMERAS_IN_USE -> "too-many-open-cameras" - CameraDevice.StateCallback.ERROR_CAMERA_DISABLED -> "camera-is-disabled-by-android" - CameraDevice.StateCallback.ERROR_CAMERA_DEVICE -> "unknown-camera-device-error" - CameraDevice.StateCallback.ERROR_CAMERA_SERVICE -> "unknown-fatal-camera-service-error" - else -> "unknown-error" - } -} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/Flash.kt b/android/src/main/java/com/mrousavy/camera/parsers/Flash.kt new file mode 100644 index 0000000000..04d5144f0e --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/Flash.kt @@ -0,0 +1,18 @@ +package com.mrousavy.camera.parsers + +enum class Flash(override val unionValue: String): JSUnionValue { + OFF("off"), + ON("on"), + AUTO("auto"); + + companion object: JSUnionValue.Companion { + override fun fromUnionValue(unionValue: String?): Flash { + return when (unionValue) { + "off" -> OFF + "on" -> ON + "auto" -> AUTO + else -> OFF + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/Format.kt b/android/src/main/java/com/mrousavy/camera/parsers/Format.kt new file mode 100644 index 0000000000..a319d3d10c --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/Format.kt @@ -0,0 +1,23 @@ +package com.mrousavy.camera.parsers + +import android.graphics.ImageFormat +import android.graphics.PixelFormat + +enum class Format(override val unionValue: String): JSUnionValue { + YUV("yuv"), + RGB("rgb"), + DNG("dng"), + UNKNOWN("unknown"); + + companion object { + fun fromImageFormat(imageFormat: Int): Format { + return when (imageFormat) { + ImageFormat.YUV_420_888 -> YUV + ImageFormat.JPEG -> RGB + PixelFormat.RGB_888 -> RGB + ImageFormat.DEPTH_JPEG -> DNG + else -> UNKNOWN + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt deleted file mode 100644 index adc4947f00..0000000000 --- a/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel+String.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.mrousavy.camera.parsers - -import android.hardware.camera2.CameraCharacteristics - -fun parseHardwareLevel(hardwareLevel: Int): String { - return when (hardwareLevel) { - CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY -> "legacy" - CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED -> "limited" - CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_EXTERNAL -> "limited" - CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL -> "full" - CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3 -> "full" - else -> "legacy" - } -} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel.kt b/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel.kt new file mode 100644 index 0000000000..fcb2491830 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/HardwareLevel.kt @@ -0,0 +1,24 @@ +package com.mrousavy.camera.parsers + +import android.hardware.camera2.CameraCharacteristics + +enum class HardwareLevel(override val unionValue: String): JSUnionValue { + LEGACY("legacy"), + LIMITED("limited"), + EXTERNAL("external"), + FULL("full"), + LEVEL_3("level-3"); + + companion object { + fun fromCameraCharacteristics(cameraCharacteristics: CameraCharacteristics): HardwareLevel { + return when (cameraCharacteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)) { + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY -> LEGACY + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED -> LIMITED + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_EXTERNAL -> EXTERNAL + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL -> FULL + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3 -> LEVEL_3 + else -> LEGACY + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt deleted file mode 100644 index 7d85829d37..0000000000 --- a/android/src/main/java/com/mrousavy/camera/parsers/ImageFormat+String.kt +++ /dev/null @@ -1,51 +0,0 @@ -package com.mrousavy.camera.parsers - -import android.graphics.ImageFormat -import android.graphics.PixelFormat - -/** - * Parses ImageFormat/PixelFormat int to a string representation useable for the TypeScript types. - */ -fun parseImageFormat(imageFormat: Int): String { - return when (imageFormat) { - ImageFormat.YUV_420_888 -> "yuv" - ImageFormat.JPEG -> "rgb" - PixelFormat.RGB_888 -> "rgb" - else -> "unknown" - /* - ImageFormat.YUV_422_888 -> "yuv" - ImageFormat.YUV_444_888 -> "yuv" - ImageFormat.JPEG -> "jpeg" - ImageFormat.DEPTH_JPEG -> "jpeg-depth" - ImageFormat.RAW_SENSOR -> "raw" - ImageFormat.RAW_PRIVATE -> "raw" - ImageFormat.HEIC -> "heic" - ImageFormat.PRIVATE -> "private" - ImageFormat.DEPTH16 -> "depth-16" - ImageFormat.UNKNOWN -> "TODOFILL" - ImageFormat.RGB_565 -> "TODOFILL" - ImageFormat.YV12 -> "TODOFILL" - ImageFormat.Y8 -> "TODOFILL" - ImageFormat.NV16 -> "TODOFILL" - ImageFormat.NV21 -> "TODOFILL" - ImageFormat.YUY2 -> "TODOFILL" - ImageFormat.FLEX_RGB_888 -> "TODOFILL" - ImageFormat.FLEX_RGBA_8888 -> "TODOFILL" - ImageFormat.RAW10 -> "TODOFILL" - ImageFormat.RAW12 -> "TODOFILL" - ImageFormat.DEPTH_POINT_CLOUD -> "TODOFILL" - @Suppress("DUPLICATE_LABEL_IN_WHEN") - PixelFormat.UNKNOWN -> "TODOFILL" - PixelFormat.TRANSPARENT -> "TODOFILL" - PixelFormat.TRANSLUCENT -> "TODOFILL" - PixelFormat.RGBX_8888 -> "TODOFILL" - PixelFormat.RGBA_F16 -> "TODOFILL" - PixelFormat.RGBA_8888 -> "TODOFILL" - PixelFormat.RGBA_1010102 -> "TODOFILL" - PixelFormat.OPAQUE -> "TODOFILL" - @Suppress("DUPLICATE_LABEL_IN_WHEN") - PixelFormat.RGB_565 -> "TODOFILL" - PixelFormat.RGB_888 -> "TODOFILL" - */ - } -} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/JSUnionValue.kt b/android/src/main/java/com/mrousavy/camera/parsers/JSUnionValue.kt new file mode 100644 index 0000000000..2d94f321ec --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/JSUnionValue.kt @@ -0,0 +1,9 @@ +package com.mrousavy.camera.parsers + +interface JSUnionValue { + val unionValue: String + + interface Companion { + fun fromUnionValue(unionValue: String?): T? + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/LensFacing.kt b/android/src/main/java/com/mrousavy/camera/parsers/LensFacing.kt new file mode 100644 index 0000000000..554a249a1c --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/LensFacing.kt @@ -0,0 +1,20 @@ +package com.mrousavy.camera.parsers + +import android.hardware.camera2.CameraCharacteristics + +enum class LensFacing(override val unionValue: String): JSUnionValue { + BACK("back"), + FRONT("front"), + EXTERNAL("external"); + + companion object { + fun fromCameraCharacteristics(cameraCharacteristics: CameraCharacteristics): LensFacing { + return when (cameraCharacteristics.get(CameraCharacteristics.LENS_FACING)!!) { + CameraCharacteristics.LENS_FACING_BACK -> BACK + CameraCharacteristics.LENS_FACING_FRONT -> FRONT + CameraCharacteristics.LENS_FACING_EXTERNAL -> EXTERNAL + else -> EXTERNAL + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/LenseFacing+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/LenseFacing+String.kt deleted file mode 100644 index 335ed6e356..0000000000 --- a/android/src/main/java/com/mrousavy/camera/parsers/LenseFacing+String.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.mrousavy.camera.parsers - -import android.hardware.camera2.CameraCharacteristics - -/** - * Parses Lens Facing int to a string representation useable for the TypeScript types. - */ -fun parseLensFacing(lensFacing: Int?): String? { - return when (lensFacing) { - CameraCharacteristics.LENS_FACING_BACK -> "back" - CameraCharacteristics.LENS_FACING_FRONT -> "front" - CameraCharacteristics.LENS_FACING_EXTERNAL -> "external" - else -> null - } -} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt b/android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt new file mode 100644 index 0000000000..5b1b863b1d --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt @@ -0,0 +1,40 @@ +package com.mrousavy.camera.parsers + +import android.view.Surface + +enum class Orientation(override val unionValue: String): JSUnionValue { + PORTRAIT("portrait"), + LANDSCAPE_RIGHT("landscape-right"), + PORTRAIT_UPSIDE_DOWN("portrait-upside-down"), + LANDSCAPE_LEFT("landscape-left"); + + fun toSurfaceRotation(): Int { + return when(this) { + PORTRAIT -> Surface.ROTATION_0 + LANDSCAPE_RIGHT -> Surface.ROTATION_90 + PORTRAIT_UPSIDE_DOWN -> Surface.ROTATION_180 + LANDSCAPE_LEFT -> Surface.ROTATION_270 + } + } + + companion object: JSUnionValue.Companion { + override fun fromUnionValue(unionValue: String?): Orientation? { + return when (unionValue) { + "portrait" -> PORTRAIT + "landscape-right" -> LANDSCAPE_RIGHT + "portrait-upside-down" -> PORTRAIT_UPSIDE_DOWN + "landscape-left" -> LANDSCAPE_LEFT + else -> null + } + } + + fun fromRotationDegrees(rotationDegrees: Int): Orientation { + return when (rotationDegrees) { + in 45..135 -> LANDSCAPE_RIGHT + in 135..225 -> PORTRAIT_UPSIDE_DOWN + in 225..315 -> LANDSCAPE_LEFT + else -> PORTRAIT + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt deleted file mode 100644 index a85a7bc605..0000000000 --- a/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt +++ /dev/null @@ -1,11 +0,0 @@ -package com.mrousavy.camera.parsers - -import android.content.pm.PackageManager - -fun parsePermissionStatus(status: Int): String { - return when (status) { - PackageManager.PERMISSION_DENIED -> "denied" - PackageManager.PERMISSION_GRANTED -> "authorized" - else -> "not-determined" - } -} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus.kt b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus.kt new file mode 100644 index 0000000000..56690345d3 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus.kt @@ -0,0 +1,19 @@ +package com.mrousavy.camera.parsers + +import android.content.pm.PackageManager + +enum class PermissionStatus(override val unionValue: String): JSUnionValue { + DENIED("denied"), + NOT_DETERMINED("not-determined"), + GRANTED("granted"); + + companion object { + fun fromPermissionStatus(status: Int): PermissionStatus { + return when (status) { + PackageManager.PERMISSION_DENIED -> DENIED + PackageManager.PERMISSION_GRANTED -> GRANTED + else -> NOT_DETERMINED + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/PreviewType.kt b/android/src/main/java/com/mrousavy/camera/parsers/PreviewType.kt new file mode 100644 index 0000000000..7ffe4d39d4 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/PreviewType.kt @@ -0,0 +1,18 @@ +package com.mrousavy.camera.parsers + +enum class PreviewType(override val unionValue: String): JSUnionValue { + NONE("none"), + NATIVE("native"), + SKIA("skia"); + + companion object: JSUnionValue.Companion { + override fun fromUnionValue(unionValue: String?): PreviewType { + return when (unionValue) { + "none" -> NONE + "native" -> NATIVE + "skia" -> SKIA + else -> NONE + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/QualityPrioritization.kt b/android/src/main/java/com/mrousavy/camera/parsers/QualityPrioritization.kt new file mode 100644 index 0000000000..2df821fa99 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/QualityPrioritization.kt @@ -0,0 +1,18 @@ +package com.mrousavy.camera.parsers + +enum class QualityPrioritization(override val unionValue: String): JSUnionValue { + SPEED("speed"), + BALANCED("balanced"), + QUALITY("quality"); + + companion object: JSUnionValue.Companion { + override fun fromUnionValue(unionValue: String?): QualityPrioritization { + return when (unionValue) { + "speed" -> SPEED + "balanced" -> BALANCED + "quality" -> QUALITY + else -> BALANCED + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/Torch.kt b/android/src/main/java/com/mrousavy/camera/parsers/Torch.kt new file mode 100644 index 0000000000..da821dbb33 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/Torch.kt @@ -0,0 +1,16 @@ +package com.mrousavy.camera.parsers + +enum class Torch(override val unionValue: String): JSUnionValue { + OFF("off"), + ON("on"); + + companion object: JSUnionValue.Companion { + override fun fromUnionValue(unionValue: String?): Torch { + return when (unionValue) { + "off" -> OFF + "on" -> ON + else -> OFF + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt deleted file mode 100644 index 171633d246..0000000000 --- a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode+String.kt +++ /dev/null @@ -1,36 +0,0 @@ -package com.mrousavy.camera.parsers - -import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_OFF -import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_ON -import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION -import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_OFF -import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_ON - -data class VideoStabilizationMode(val digitalMode: Int, - val opticalMode: Int) - -fun parseDigitalVideoStabilizationMode(stabiliazionMode: Int): String { - return when (stabiliazionMode) { - CONTROL_VIDEO_STABILIZATION_MODE_OFF -> "off" - CONTROL_VIDEO_STABILIZATION_MODE_ON -> "standard" - CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION -> "cinematic" - else -> "off" - } -} -fun parseOpticalVideoStabilizationMode(stabiliazionMode: Int): String { - return when (stabiliazionMode) { - LENS_OPTICAL_STABILIZATION_MODE_OFF -> "off" - LENS_OPTICAL_STABILIZATION_MODE_ON -> "cinematic-extended" - else -> "off" - } -} - -fun getVideoStabilizationMode(stabiliazionMode: String): VideoStabilizationMode { - return when (stabiliazionMode) { - "off" -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_OFF, LENS_OPTICAL_STABILIZATION_MODE_OFF) - "standard" -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_ON, LENS_OPTICAL_STABILIZATION_MODE_OFF) - "cinematic" -> VideoStabilizationMode(2 /* CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION */, LENS_OPTICAL_STABILIZATION_MODE_OFF) - "cinematic-extended" -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_OFF, LENS_OPTICAL_STABILIZATION_MODE_ON) - else -> VideoStabilizationMode(CONTROL_VIDEO_STABILIZATION_MODE_OFF, LENS_OPTICAL_STABILIZATION_MODE_OFF) - } -} diff --git a/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode.kt b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode.kt new file mode 100644 index 0000000000..59aeedae0e --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/VideoStabilizationMode.kt @@ -0,0 +1,59 @@ +package com.mrousavy.camera.parsers + +import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_OFF +import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_ON +import android.hardware.camera2.CameraMetadata.CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION +import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_OFF +import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_ON + +enum class VideoStabilizationMode(override val unionValue: String): JSUnionValue { + OFF("off"), + STANDARD("standard"), + CINEMATIC("cinematic"), + CINEMATIC_EXTENDED("cinematic-extended"); + + fun toDigitalStabilizationMode(): Int { + return when (this) { + OFF -> CONTROL_VIDEO_STABILIZATION_MODE_OFF + STANDARD -> CONTROL_VIDEO_STABILIZATION_MODE_ON + CINEMATIC -> 2 /* CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION */ + else -> CONTROL_VIDEO_STABILIZATION_MODE_OFF + } + } + + fun toOpticalStabilizationMode(): Int { + return when (this) { + OFF -> LENS_OPTICAL_STABILIZATION_MODE_OFF + CINEMATIC_EXTENDED -> LENS_OPTICAL_STABILIZATION_MODE_ON + else -> LENS_OPTICAL_STABILIZATION_MODE_OFF + } + } + + companion object: JSUnionValue.Companion { + override fun fromUnionValue(unionValue: String?): VideoStabilizationMode? { + return when (unionValue) { + "off" -> OFF + "standard" -> STANDARD + "cinematic" -> CINEMATIC + "cinematic-extended" -> CINEMATIC_EXTENDED + else -> null + } + } + + fun fromDigitalVideoStabilizationMode(stabiliazionMode: Int): VideoStabilizationMode { + return when (stabiliazionMode) { + CONTROL_VIDEO_STABILIZATION_MODE_OFF -> OFF + CONTROL_VIDEO_STABILIZATION_MODE_ON -> STANDARD + CONTROL_VIDEO_STABILIZATION_MODE_PREVIEW_STABILIZATION -> CINEMATIC + else -> OFF + } + } + fun fromOpticalVideoStabilizationMode(stabiliazionMode: Int): VideoStabilizationMode { + return when (stabiliazionMode) { + LENS_OPTICAL_STABILIZATION_MODE_OFF -> OFF + LENS_OPTICAL_STABILIZATION_MODE_ON -> CINEMATIC_EXTENDED + else -> OFF + } + } + } +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt index 43359c3e5a..39ce540b38 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraDeviceDetails.kt @@ -13,17 +13,16 @@ import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import com.mrousavy.camera.extensions.bigger -import com.mrousavy.camera.parsers.parseDigitalVideoStabilizationMode -import com.mrousavy.camera.parsers.parseHardwareLevel -import com.mrousavy.camera.parsers.parseImageFormat -import com.mrousavy.camera.parsers.parseLensFacing -import com.mrousavy.camera.parsers.parseOpticalVideoStabilizationMode +import com.mrousavy.camera.parsers.Format +import com.mrousavy.camera.parsers.HardwareLevel +import com.mrousavy.camera.parsers.LensFacing +import com.mrousavy.camera.parsers.VideoStabilizationMode import kotlin.math.PI import kotlin.math.atan class CameraDeviceDetails(private val cameraManager: CameraManager, private val cameraId: String) { private val characteristics = cameraManager.getCameraCharacteristics(cameraId) - private val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) ?: CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY + private val hardwareLevel = HardwareLevel.fromCameraCharacteristics(characteristics) private val capabilities = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES) ?: IntArray(0) private val extensions = getSupportedExtensions() @@ -32,12 +31,12 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private val supportsDepthCapture = capabilities.contains(8 /* TODO: CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT */) private val supportsRawCapture = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_RAW) private val supportsLowLightBoost = extensions.contains(4 /* TODO: CameraExtensionCharacteristics.EXTENSION_NIGHT */) - private val lensFacing = characteristics.get(CameraCharacteristics.LENS_FACING)!! + private val lensFacing = LensFacing.fromCameraCharacteristics(characteristics) private val hasFlash = characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE) ?: false private val focalLengths = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) ?: floatArrayOf(35f /* 35mm default */) private val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! private val name = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) - else null) ?: "${parseLensFacing(lensFacing)} (${cameraId})" + else null) ?: "$lensFacing (${cameraId})" // "formats" (all possible configurations for this device) private val zoomRange = (if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) characteristics.get(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE) @@ -53,7 +52,6 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private val supportsVideoHdr = getHasVideoHdr() private val videoFormat = ImageFormat.YUV_420_888 - private val imageFormat = ImageFormat.JPEG // get extensions (HDR, Night Mode, ..) private fun getSupportedExtensions(): List { @@ -80,10 +78,12 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val private fun createStabilizationModes(): ReadableArray { val array = Arguments.createArray() digitalStabilizationModes.forEach { videoStabilizationMode -> - array.pushString(parseDigitalVideoStabilizationMode(videoStabilizationMode)) + val mode = VideoStabilizationMode.fromDigitalVideoStabilizationMode(videoStabilizationMode) + array.pushString(mode.unionValue) } opticalStabilizationModes.forEach { videoStabilizationMode -> - array.pushString(parseOpticalVideoStabilizationMode(videoStabilizationMode)) + val mode = VideoStabilizationMode.fromOpticalVideoStabilizationMode(videoStabilizationMode) + array.pushString(mode.unionValue) } return array } @@ -122,7 +122,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val return 2 * atan(sensorSize.bigger / (focalLengths[0] * 2)) * (180 / PI) } - private fun buildFormatMap(photoSize: Size, videoSize: Size, outputFormat: Int, fpsRange: Range): ReadableMap { + private fun buildFormatMap(photoSize: Size, videoSize: Size, outputFormat: Format, fpsRange: Range): ReadableMap { val map = Arguments.createMap() map.putInt("photoHeight", photoSize.height) map.putInt("photoWidth", photoSize.width) @@ -137,7 +137,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val map.putBoolean("supportsPhotoHDR", supportsPhotoHdr) map.putString("autoFocusSystem", "contrast-detection") // TODO: Is this wrong? map.putArray("videoStabilizationModes", createStabilizationModes()) - map.putString("pixelFormat", parseImageFormat(outputFormat)) + map.putString("pixelFormat", outputFormat.unionValue) return map } @@ -165,7 +165,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val val maxFps = (1.0 / (frameDuration.toDouble() / 1000000000)).toInt() photoSizes.forEach { photoSize -> - val map = buildFormatMap(photoSize, videoSize, videoFormat, Range(1, maxFps)) + val map = buildFormatMap(photoSize, videoSize, Format.fromImageFormat(videoFormat), Range(1, maxFps)) array.pushMap(map) } } @@ -180,7 +180,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val val map = Arguments.createMap() map.putString("id", cameraId) map.putArray("devices", getDeviceTypes()) - map.putString("position", parseLensFacing(lensFacing)) + map.putString("position", lensFacing.unionValue) map.putString("name", name) map.putBoolean("hasFlash", hasFlash) map.putBoolean("hasTorch", hasFlash) @@ -192,7 +192,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val map.putDouble("minZoom", minZoom) map.putDouble("maxZoom", maxZoom) map.putDouble("neutralZoom", 1.0) // Zoom is always relative to 1.0 on Android - map.putString("hardwareLevel", parseHardwareLevel(hardwareLevel)) + map.putString("hardwareLevel", hardwareLevel.unionValue) val array = Arguments.createArray() cameraConfig.outputFormats.forEach { f -> diff --git a/docs/docs/guides/MOCKING.mdx b/docs/docs/guides/MOCKING.mdx index 14f5f890c9..2f4944dc0c 100644 --- a/docs/docs/guides/MOCKING.mdx +++ b/docs/docs/guides/MOCKING.mdx @@ -39,69 +39,69 @@ module.exports = { ### Create proxy for original and mocked modules - 1. Create a new folder `vision-camera` anywhere in your project. - 2. Inside that folder, create `vision-camera.js` and `vision-camera.e2e.js`. - 3. Inside `vision-camera.js`, export the original react native modules you need to mock, and - inside `vision-camera.e2e.js` export the mocked modules. +1. Create a new folder `vision-camera` anywhere in your project. +2. Inside that folder, create `vision-camera.js` and `vision-camera.e2e.js`. +3. Inside `vision-camera.js`, export the original react native modules you need to mock, and +inside `vision-camera.e2e.js` export the mocked modules. - In this example, several functions of the modules `Camera` and `sortDevices` are mocked. - Define your mocks following the [original definitions](https://github.com/mrousavy/react-native-vision-camera/tree/main/src). +In this example, several functions of the modules `Camera` and `sortDevices` are mocked. +Define your mocks following the [original definitions](https://github.com/mrousavy/react-native-vision-camera/tree/main/src). - ```js - // vision-camera.js +```js +// vision-camera.js - import { Camera, sortDevices } from 'react-native-vision-camera'; +import { Camera, sortDevices } from 'react-native-vision-camera'; - export const VisionCamera = Camera; - export const visionCameraSortDevices = sortDevices; - ``` +export const VisionCamera = Camera; +export const visionCameraSortDevices = sortDevices; +``` - ```js - // vision-camera.e2e.js +```js +// vision-camera.e2e.js - import React from 'react'; - import RNFS, { writeFile } from 'react-native-fs'; +import React from 'react'; +import RNFS, { writeFile } from 'react-native-fs'; - console.log('[DETOX] Using mocked react-native-vision-camera'); +console.log('[DETOX] Using mocked react-native-vision-camera'); - export class VisionCamera extends React.PureComponent { - static async getAvailableCameraDevices() { - return ( - [ - { - position: 'back', - }, - ] - ); - } +export class VisionCamera extends React.PureComponent { + static async getAvailableCameraDevices() { + return ( + [ + { + position: 'back', + }, + ] + ); + } - static async getCameraPermissionStatus() { - return 'authorized'; - } + static async getCameraPermissionStatus() { + return 'granted'; + } - static async requestCameraPermission() { - return 'authorized'; - } + static async requestCameraPermission() { + return 'granted'; + } - async takePhoto() { - const writePath = `${RNFS.DocumentDirectoryPath}/simulated_camera_photo.png`; + async takePhoto() { + const writePath = `${RNFS.DocumentDirectoryPath}/simulated_camera_photo.png`; - const imageDataBase64 = 'some_large_base_64_encoded_simulated_camera_photo'; - await writeFile(writePath, imageDataBase64, 'base64'); + const imageDataBase64 = 'some_large_base_64_encoded_simulated_camera_photo'; + await writeFile(writePath, imageDataBase64, 'base64'); - return { path: writePath }; - } + return { path: writePath }; + } - render() { - return null; - } - } + render() { + return null; + } +} - export const visionCameraSortDevices = (_left, _right) => 1; - ``` +export const visionCameraSortDevices = (_left, _right) => 1; +``` - These mocked modules allows us to get authorized camera permissions, get one back camera - available and take a fake photo, while the component doesn't render when instantiated. +These mocked modules allows us to get granted camera permissions, get one back camera +available and take a fake photo, while the component doesn't render when instantiated. ### Use proxy module diff --git a/docs/docs/guides/SETUP.mdx b/docs/docs/guides/SETUP.mdx index 8ff0ca403d..c4901d68df 100644 --- a/docs/docs/guides/SETUP.mdx +++ b/docs/docs/guides/SETUP.mdx @@ -138,7 +138,7 @@ const microphonePermission = await Camera.getMicrophonePermissionStatus() A permission status can have the following values: -* `authorized`: Your app is authorized to use said permission. Continue with [**using the `` view**](#use-the-camera-view). +* `granted`: Your app is authorized to use said permission. Continue with [**using the `` view**](#use-the-camera-view). * `not-determined`: Your app has not yet requested permission from the user. [Continue by calling the **request** functions.](#requesting-permissions) * `denied`: Your app has already requested permissions from the user, but was explicitly denied. You cannot use the **request** functions again, but you can use the [`Linking` API](https://reactnative.dev/docs/linking#opensettings) to redirect the user to the Settings App where he can manually grant the permission. * `restricted`: (iOS only) Your app cannot use the Camera or Microphone because that functionality has been restricted, possibly due to active restrictions such as parental controls being in place. @@ -158,7 +158,7 @@ const newMicrophonePermission = await Camera.requestMicrophonePermission() The permission request status can have the following values: -* `authorized`: Your app is authorized to use said permission. Continue with [**using the `` view**](#use-the-camera-view). +* `granted`: Your app is authorized to use said permission. Continue with [**using the `` view**](#use-the-camera-view). * `denied`: The user explicitly denied the permission request alert. You cannot use the **request** functions again, but you can use the [`Linking` API](https://reactnative.dev/docs/linking#opensettings) to redirect the user to the Settings App where he can manually grant the permission. * `restricted`: (iOS only) Your app cannot use the Camera or Microphone because that functionality has been restricted, possibly due to active restrictions such as parental controls being in place. diff --git a/example/src/App.tsx b/example/src/App.tsx index 0cfc76451a..5cfd958edd 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -26,7 +26,7 @@ export function App(): React.ReactElement | null { return null; } - const showPermissionsPage = cameraPermission !== 'authorized' || microphonePermission === 'not-determined'; + const showPermissionsPage = cameraPermission !== 'granted' || microphonePermission === 'not-determined'; return ( diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index 847628b7ec..ca22e0d464 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -169,7 +169,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { }, [neutralZoom, zoom]); useEffect(() => { - Camera.getMicrophonePermissionStatus().then((status) => setHasMicrophonePermission(status === 'authorized')); + Camera.getMicrophonePermissionStatus().then((status) => setHasMicrophonePermission(status === 'granted')); }, []); //#endregion diff --git a/example/src/PermissionsPage.tsx b/example/src/PermissionsPage.tsx index b2e7af64b7..4cdc072992 100644 --- a/example/src/PermissionsPage.tsx +++ b/example/src/PermissionsPage.tsx @@ -34,7 +34,7 @@ export function PermissionsPage({ navigation }: Props): React.ReactElement { }, []); useEffect(() => { - if (cameraPermissionStatus === 'authorized' && microphonePermissionStatus === 'authorized') navigation.replace('CameraPage'); + if (cameraPermissionStatus === 'granted' && microphonePermissionStatus === 'granted') navigation.replace('CameraPage'); }, [cameraPermissionStatus, microphonePermissionStatus, navigation]); return ( @@ -42,7 +42,7 @@ export function PermissionsPage({ navigation }: Props): React.ReactElement { Welcome to{'\n'}Vision Camera. - {cameraPermissionStatus !== 'authorized' && ( + {cameraPermissionStatus !== 'granted' && ( Vision Camera needs Camera permission.{' '} @@ -50,7 +50,7 @@ export function PermissionsPage({ navigation }: Props): React.ReactElement { )} - {microphonePermissionStatus !== 'authorized' && ( + {microphonePermissionStatus !== 'granted' && ( Vision Camera needs Microphone permission.{' '} diff --git a/ios/Parsers/AVAuthorizationStatus+descriptor.swift b/ios/Parsers/AVAuthorizationStatus+descriptor.swift index e421a38c94..26ad9a9bf9 100644 --- a/ios/Parsers/AVAuthorizationStatus+descriptor.swift +++ b/ios/Parsers/AVAuthorizationStatus+descriptor.swift @@ -12,7 +12,7 @@ extension AVAuthorizationStatus { var descriptor: String { switch self { case .authorized: - return "authorized" + return "granted" case .denied: return "denied" case .notDetermined: diff --git a/src/Camera.tsx b/src/Camera.tsx index 8ed4846f79..e737006d52 100644 --- a/src/Camera.tsx +++ b/src/Camera.tsx @@ -13,8 +13,8 @@ import type { CameraVideoCodec, RecordVideoOptions, VideoFile, VideoFileType } f import { VisionCameraProxy } from './FrameProcessorPlugins'; //#region Types -export type CameraPermissionStatus = 'authorized' | 'not-determined' | 'denied' | 'restricted'; -export type CameraPermissionRequestResult = 'authorized' | 'denied'; +export type CameraPermissionStatus = 'granted' | 'not-determined' | 'denied' | 'restricted'; +export type CameraPermissionRequestResult = 'granted' | 'denied'; interface OnErrorEvent { code: string; From 1d2241f09ffc5a93226906a7f9177c9bf5a1a7cd Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 8 Aug 2023 11:04:11 +0200 Subject: [PATCH 074/180] Implement Orientation (I think) --- .../java/com/mrousavy/camera/CameraSession.kt | 25 +++++++++++++---- .../mrousavy/camera/CameraView+TakePhoto.kt | 4 ++- .../java/com/mrousavy/camera/CameraView.kt | 11 ++++---- .../CameraDevice+createPhotoCaptureRequest.kt | 9 ++++-- .../mrousavy/camera/parsers/Orientation.kt | 28 +++++++++++++++---- 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 4ccd4b1e6a..b2d5b2fa8a 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -1,6 +1,7 @@ package com.mrousavy.camera import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest @@ -18,6 +19,7 @@ import com.mrousavy.camera.extensions.openCamera import com.mrousavy.camera.extensions.tryClose import com.mrousavy.camera.parsers.CameraDeviceError import com.mrousavy.camera.parsers.Flash +import com.mrousavy.camera.parsers.Orientation import com.mrousavy.camera.parsers.QualityPrioritization import com.mrousavy.camera.parsers.VideoStabilizationMode import com.mrousavy.camera.utils.CameraOutputs @@ -42,7 +44,7 @@ class CameraSession(private val cameraManager: CameraManager, data class CapturedPhoto(val image: Image, val metadata: TotalCaptureResult, - val orientation: Int, + val orientation: Orientation, val format: Int): Closeable { override fun close() { image.close() @@ -85,6 +87,14 @@ class CameraSession(private val cameraManager: CameraManager, isRunning = false } + val orientation: Orientation + get() { + val cameraId = cameraId ?: return Orientation.PORTRAIT + val characteristics = cameraManager.getCameraCharacteristics(cameraId) + val sensorRotation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 + return Orientation.fromRotationDegrees(sensorRotation) + } + /** * Set the Camera to be used as an input device. * Calling this with the same ID twice will not re-open the Camera device. @@ -156,28 +166,31 @@ class CameraSession(private val cameraManager: CameraManager, suspend fun takePhoto(qualityPrioritization: QualityPrioritization, flashMode: Flash, enableRedEyeReduction: Boolean, - enableAutoStabilization: Boolean): CapturedPhoto { + enableAutoStabilization: Boolean, + outputOrientation: Orientation): CapturedPhoto { val captureSession = captureSession ?: throw CameraNotReadyError() val outputs = outputs ?: throw CameraNotReadyError() val photoOutput = outputs.photoOutput ?: throw PhotoNotEnabledError() + + val cameraCharacteristics = cameraManager.getCameraCharacteristics(captureSession.device.id) + val orientation = outputOrientation.toSensorRelativeOrientation(cameraCharacteristics) val captureRequest = captureSession.device.createPhotoCaptureRequest(cameraManager, photoOutput.surface, qualityPrioritization, flashMode, enableRedEyeReduction, - enableAutoStabilization) + enableAutoStabilization, + orientation) Log.i(TAG, "Photo capture 0/2 - starting capture...") val result = captureSession.capture(captureRequest) val timestamp = result[CaptureResult.SENSOR_TIMESTAMP]!! Log.i(TAG, "Photo capture 1/2 complete - received metadata with timestamp $timestamp") try { val image = photoOutputSynchronizer.await(timestamp) - // TODO: Correctly get rotationDegrees and isMirrored - val rotation = ExifUtils.computeExifOrientation(0, false) Log.i(TAG, "Photo capture 2/2 complete - received ${image.width} x ${image.height} image.") - return CapturedPhoto(image, result, rotation, image.format) + return CapturedPhoto(image, result, orientation, image.format) } catch (e: CancellationException) { throw CaptureAbortedError(false) } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt index 2c8872fac7..45d28aad5c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt @@ -34,7 +34,8 @@ suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = corouti val photo = cameraSession.takePhoto(qualityPrioritizationMode, flashMode, enableAutoRedEyeReduction, - enableAutoStabilization) + enableAutoStabilization, + outputOrientation) photo.use { Log.i(TAG, "Successfully captured ${photo.image.width} x ${photo.image.height} photo!") @@ -49,6 +50,7 @@ suspend fun CameraView.takePhoto(optionsMap: ReadableMap): WritableMap = corouti map.putString("path", path) map.putInt("width", photo.image.width) map.putInt("height", photo.image.height) + map.putString("orientation", photo.orientation.unionValue) map.putBoolean("isRawPhoto", photo.format == ImageFormat.RAW_SENSOR) // TODO: Add metadata prop to resulting photo diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index a3e0499ec1..0df68f73d9 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -14,7 +14,6 @@ import android.widget.FrameLayout import androidx.core.content.ContextCompat import com.facebook.react.bridge.ReadableMap import com.mrousavy.camera.extensions.containsAny -import com.mrousavy.camera.extensions.displayRotation import com.mrousavy.camera.extensions.installHierarchyFitter import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor @@ -98,10 +97,10 @@ class CameraView(context: Context) : FrameLayout(context) { var frameProcessor: FrameProcessor? = null - private val inputRotation: Int - get() = context.displayRotation - private val outputRotation: Int - get() = orientation?.toSurfaceRotation() ?: inputRotation + private val inputOrientation: Orientation + get() = cameraSession.orientation + internal val outputOrientation: Orientation + get() = orientation ?: inputOrientation private var minZoom: Float = 1f private var maxZoom: Float = 1f @@ -218,7 +217,7 @@ class CameraView(context: Context) : FrameLayout(context) { } else null val videoOutput = if (video == true) { CameraOutputs.VideoOutput({ image -> - val frame = Frame(image, System.currentTimeMillis(), Orientation.fromRotationDegrees(inputRotation), false) + val frame = Frame(image, System.currentTimeMillis(), outputOrientation, false) onFrame(frame) }, targetVideoSize) } else null diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt index 6e888f0bd4..877fcddeb7 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createPhotoCaptureRequest.kt @@ -6,6 +6,7 @@ import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.view.Surface import com.mrousavy.camera.parsers.Flash +import com.mrousavy.camera.parsers.Orientation import com.mrousavy.camera.parsers.QualityPrioritization fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, @@ -13,7 +14,10 @@ fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, qualityPrioritization: QualityPrioritization, flashMode: Flash, enableRedEyeReduction: Boolean, - enableAutoStabilization: Boolean): CaptureRequest { + enableAutoStabilization: Boolean, + orientation: Orientation): CaptureRequest { + val cameraCharacteristics = cameraManager.getCameraCharacteristics(this.id) + val captureRequest = when (qualityPrioritization) { // If speed, use snapshot template for fast capture QualityPrioritization.SPEED -> this.createCaptureRequest(CameraDevice.TEMPLATE_VIDEO_SNAPSHOT) @@ -29,7 +33,7 @@ fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, } captureRequest[CaptureRequest.JPEG_QUALITY] = jpegQuality.toByte() - // TODO: CaptureRequest.JPEG_ORIENTATION maybe? + captureRequest.set(CaptureRequest.JPEG_ORIENTATION, orientation.toDegrees()) when (flashMode) { // Set the Flash Mode @@ -51,7 +55,6 @@ fun CameraDevice.createPhotoCaptureRequest(cameraManager: CameraManager, } if (enableAutoStabilization) { - val cameraCharacteristics = cameraManager.getCameraCharacteristics(this.id) // Enable optical or digital image stabilization val digitalStabilization = cameraCharacteristics.get(CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES) val hasDigitalStabilization = digitalStabilization?.contains(CameraCharacteristics.CONTROL_VIDEO_STABILIZATION_MODE_ON) ?: false diff --git a/android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt b/android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt index 5b1b863b1d..9e3db44835 100644 --- a/android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt +++ b/android/src/main/java/com/mrousavy/camera/parsers/Orientation.kt @@ -1,6 +1,6 @@ package com.mrousavy.camera.parsers -import android.view.Surface +import android.hardware.camera2.CameraCharacteristics enum class Orientation(override val unionValue: String): JSUnionValue { PORTRAIT("portrait"), @@ -8,15 +8,31 @@ enum class Orientation(override val unionValue: String): JSUnionValue { PORTRAIT_UPSIDE_DOWN("portrait-upside-down"), LANDSCAPE_LEFT("landscape-left"); - fun toSurfaceRotation(): Int { + fun toDegrees(): Int { return when(this) { - PORTRAIT -> Surface.ROTATION_0 - LANDSCAPE_RIGHT -> Surface.ROTATION_90 - PORTRAIT_UPSIDE_DOWN -> Surface.ROTATION_180 - LANDSCAPE_LEFT -> Surface.ROTATION_270 + PORTRAIT -> 0 + LANDSCAPE_RIGHT -> 90 + PORTRAIT_UPSIDE_DOWN -> 180 + LANDSCAPE_LEFT -> 270 } } + fun toSensorRelativeOrientation(cameraCharacteristics: CameraCharacteristics): Orientation { + val sensorOrientation = cameraCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION)!! + + // Convert target orientation to rotation degrees (0, 90, 180, 270) + var rotationDegrees = this.toDegrees() + + // Reverse device orientation for front-facing cameras + val facingFront = cameraCharacteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT + if (facingFront) rotationDegrees = -rotationDegrees + + // Rotate sensor rotation by target rotation + val newRotationDegrees = (sensorOrientation + rotationDegrees + 360) % 360 + + return fromRotationDegrees(newRotationDegrees) + } + companion object: JSUnionValue.Companion { override fun fromUnionValue(unionValue: String?): Orientation? { return when (unionValue) { From 5355f0f2872c13cbd1eafd172e3c8189fe873436 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 8 Aug 2023 12:31:28 +0200 Subject: [PATCH 075/180] Move RefCount management to Java (Frame) --- android/src/main/cpp/FrameHostObject.cpp | 13 ++++------- android/src/main/cpp/FrameHostObject.h | 4 ---- android/src/main/cpp/java-bindings/JFrame.cpp | 10 +++++++++ android/src/main/cpp/java-bindings/JFrame.h | 2 ++ .../mrousavy/camera/CameraView+RecordVideo.kt | 4 +++- .../mrousavy/camera/frameprocessor/Frame.java | 22 +++++++++++++++++++ 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/android/src/main/cpp/FrameHostObject.cpp b/android/src/main/cpp/FrameHostObject.cpp index b0ca2d5f1f..d4db57c846 100644 --- a/android/src/main/cpp/FrameHostObject.cpp +++ b/android/src/main/cpp/FrameHostObject.cpp @@ -18,7 +18,7 @@ namespace vision { using namespace facebook; -FrameHostObject::FrameHostObject(const jni::alias_ref& frame): frame(make_global(frame)), _refCount(0) { } +FrameHostObject::FrameHostObject(const jni::alias_ref& frame): frame(make_global(frame)) { } FrameHostObject::~FrameHostObject() { // Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread @@ -95,8 +95,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr if (name == "incrementRefCount") { auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA { // Increment retain count by one. - std::lock_guard lock(this->_refCountMutex); - this->_refCount++; + this->frame->incrementRefCount(); return jsi::Value::undefined(); }; return jsi::Function::createFromHostFunction(runtime, @@ -107,12 +106,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr if (name == "decrementRefCount") { auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA { - // Decrement retain count by one. If the retain count is zero, we close the Frame. - std::lock_guard lock(this->_refCountMutex); - this->_refCount--; - if (_refCount < 1) { - this->frame->close(); - } + // Decrement retain count by one. If the retain count is zero, the Frame gets closed. + this->frame->decrementRefCount(); return jsi::Value::undefined(); }; return jsi::Function::createFromHostFunction(runtime, diff --git a/android/src/main/cpp/FrameHostObject.h b/android/src/main/cpp/FrameHostObject.h index fc4f5214a2..7ce7e3091f 100644 --- a/android/src/main/cpp/FrameHostObject.h +++ b/android/src/main/cpp/FrameHostObject.h @@ -9,7 +9,6 @@ #include #include #include -#include #include "java-bindings/JFrame.h" @@ -31,9 +30,6 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject { private: static auto constexpr TAG = "VisionCamera"; - - size_t _refCount; - std::mutex _refCountMutex; }; } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrame.cpp b/android/src/main/cpp/java-bindings/JFrame.cpp index dce59e0a08..393a47a3b6 100644 --- a/android/src/main/cpp/java-bindings/JFrame.cpp +++ b/android/src/main/cpp/java-bindings/JFrame.cpp @@ -62,6 +62,16 @@ local_ref JFrame::toByteArray() const { return toByteArrayMethod(self()); } +void JFrame::incrementRefCount() { + static const auto incrementRefCountMethod = getClass()->getMethod("incrementRefCount"); + incrementRefCountMethod(self()); +} + +void JFrame::decrementRefCount() { + static const auto decrementRefCountMethod = getClass()->getMethod("decrementRefCount"); + decrementRefCountMethod(self()); +} + void JFrame::close() { static const auto closeMethod = getClass()->getMethod("close"); closeMethod(self()); diff --git a/android/src/main/cpp/java-bindings/JFrame.h b/android/src/main/cpp/java-bindings/JFrame.h index ac6e43c94b..b419fe3b97 100644 --- a/android/src/main/cpp/java-bindings/JFrame.h +++ b/android/src/main/cpp/java-bindings/JFrame.h @@ -26,6 +26,8 @@ struct JFrame : public JavaClass { local_ref getOrientation() const; local_ref getPixelFormat() const; local_ref toByteArray() const; + void incrementRefCount(); + void decrementRefCount(); void close(); }; diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt index 5997377098..4d7a4bdf1e 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt @@ -48,10 +48,12 @@ fun CameraView.stopRecording() { } fun CameraView.onFrame(frame: Frame) { + frame.incrementRefCount() + Log.d(CameraView.TAG, "New Frame available!") if (frameProcessor != null) { frameProcessor?.call(frame) } - // TODO: Record Video here using MediaCodec + frame.decrementRefCount() } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index 233eec1969..4da794f112 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -7,12 +7,14 @@ import com.mrousavy.camera.parsers.Orientation; import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; public class Frame { private final Image image; private final boolean isMirrored; private final long timestamp; private final Orientation orientation; + private int refCount = 0; public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) { this.image = image; @@ -113,6 +115,26 @@ public byte[] toByteArray() { } } + @SuppressWarnings("unused") + @DoNotStrip + public void incrementRefCount() { + synchronized (this) { + refCount++; + } + } + + @SuppressWarnings("unused") + @DoNotStrip + public void decrementRefCount() { + synchronized (this) { + refCount--; + if (refCount <= 0) { + // If no reference is held on this Image, close it. + image.close(); + } + } + } + @SuppressWarnings("unused") @DoNotStrip private void close() { From 49c3079b1d6fc24860aa1aa6167366fd7bfaf883 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 8 Aug 2023 12:31:37 +0200 Subject: [PATCH 076/180] Don't crash when dropping a Frame --- .../main/java/com/mrousavy/camera/utils/CameraOutputs.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt b/android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt index 2ded6847ec..640632554e 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/CameraOutputs.kt @@ -14,6 +14,7 @@ import com.mrousavy.camera.extensions.OutputType import com.mrousavy.camera.extensions.SurfaceOutput import com.mrousavy.camera.extensions.closestToOrMax import java.io.Closeable +import java.lang.IllegalStateException class CameraOutputs(val cameraId: String, cameraManager: CameraManager, @@ -123,8 +124,12 @@ class CameraOutputs(val cameraId: String, val imageReader = ImageReader.newInstance(size.width, size.height, video.format, VIDEO_OUTPUT_BUFFER_SIZE) imageReader.setOnImageAvailableListener({ reader -> - val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener - video.onFrame(image) + try { + val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener + video.onFrame(image) + } catch (e: IllegalStateException) { + Log.e(TAG, "Failed to acquire a new Image, dropping a Frame.. The Frame Processor cannot keep up with the Camera's FPS!", e) + } }, CameraQueues.videoQueue.handler) Log.i(TAG, "Adding ${size.width}x${size.height} video output. (Format: $video.format)") From 290ae45c4ab03420f3d36662d9fa6c1b0ef0315e Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 8 Aug 2023 12:31:46 +0200 Subject: [PATCH 077/180] Prefer Devices with higher max resolution --- src/utils/FormatFilter.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/utils/FormatFilter.ts b/src/utils/FormatFilter.ts index 70576b9d71..f44b0d866b 100644 --- a/src/utils/FormatFilter.ts +++ b/src/utils/FormatFilter.ts @@ -24,6 +24,17 @@ export const sortDevices = (left: CameraDevice, right: CameraDevice): number => if (leftHasWideAngle) leftPoints += 2; if (rightHasWideAngle) rightPoints += 2; + const leftMaxResolution = left.formats.reduce( + (prev, curr) => Math.max(prev, curr.videoHeight * curr.videoWidth + curr.photoHeight * curr.photoWidth), + 0, + ); + const rightMaxResolution = right.formats.reduce( + (prev, curr) => Math.max(prev, curr.videoHeight * curr.videoWidth + curr.photoHeight * curr.photoWidth), + 0, + ); + if (leftMaxResolution > rightMaxResolution) leftPoints += 3; + if (rightMaxResolution > leftMaxResolution) rightPoints += 3; + // telephoto cameras often have very poor quality. const leftHasTelephoto = left.devices.includes('telephoto-camera'); const rightHasTelephoto = right.devices.includes('telephoto-camera'); From fb0e02cbb94635aeb42329b516c68ecf20625a7e Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 8 Aug 2023 12:41:57 +0200 Subject: [PATCH 078/180] Prefer multi-cams --- src/utils/FormatFilter.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/FormatFilter.ts b/src/utils/FormatFilter.ts index f44b0d866b..0a11bda4ab 100644 --- a/src/utils/FormatFilter.ts +++ b/src/utils/FormatFilter.ts @@ -24,6 +24,9 @@ export const sortDevices = (left: CameraDevice, right: CameraDevice): number => if (leftHasWideAngle) leftPoints += 2; if (rightHasWideAngle) rightPoints += 2; + if (left.isMultiCam) leftPoints += 2; + if (right.isMultiCam) rightPoints += 2; + const leftMaxResolution = left.formats.reduce( (prev, curr) => Math.max(prev, curr.videoHeight * curr.videoWidth + curr.photoHeight * curr.photoWidth), 0, From a7b5d45f474e63c8b17ad6219d24c85641cfa56f Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 8 Aug 2023 12:45:22 +0200 Subject: [PATCH 079/180] Use FastImage for Media Page --- example/package.json | 1 + example/src/MediaPage.tsx | 14 ++++++-------- example/yarn.lock | 5 +++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/example/package.json b/example/package.json index be4f8d0aea..4399ac4b92 100644 --- a/example/package.json +++ b/example/package.json @@ -21,6 +21,7 @@ "@shopify/react-native-skia": "^0.1.197", "react": "^18.2.0", "react-native": "^0.72.3", + "react-native-fast-image": "^8.6.3", "react-native-gesture-handler": "^2.12.1", "react-native-pressable-opacity": "^1.0.10", "react-native-reanimated": "^3.4.1", diff --git a/example/src/MediaPage.tsx b/example/src/MediaPage.tsx index 11996725b9..f646abc32f 100644 --- a/example/src/MediaPage.tsx +++ b/example/src/MediaPage.tsx @@ -1,5 +1,5 @@ import React, { useCallback, useMemo, useState } from 'react'; -import { StyleSheet, View, Image, ActivityIndicator, PermissionsAndroid, Platform } from 'react-native'; +import { StyleSheet, View, ActivityIndicator, PermissionsAndroid, Platform } from 'react-native'; import Video, { LoadError, OnLoadData } from 'react-native-video'; import { SAFE_AREA_PADDING } from './Constants'; import { useIsForeground } from './hooks/useIsForeground'; @@ -8,11 +8,10 @@ import IonIcon from 'react-native-vector-icons/Ionicons'; import { Alert } from 'react-native'; import { CameraRoll } from '@react-native-camera-roll/camera-roll'; import { StatusBarBlurBackground } from './views/StatusBarBlurBackground'; -import type { NativeSyntheticEvent } from 'react-native'; -import type { ImageLoadEventData } from 'react-native'; import type { NativeStackScreenProps } from '@react-navigation/native-stack'; import type { Routes } from './Routes'; import { useIsFocused } from '@react-navigation/core'; +import FastImage, { OnLoadEvent } from 'react-native-fast-image'; const requestSavePermission = async (): Promise => { if (Platform.OS !== 'android') return true; @@ -27,8 +26,7 @@ const requestSavePermission = async (): Promise => { return hasPermission; }; -const isVideoOnLoadEvent = (event: OnLoadData | NativeSyntheticEvent): event is OnLoadData => - 'duration' in event && 'naturalSize' in event; +const isVideoOnLoadEvent = (event: OnLoadData | OnLoadEvent): event is OnLoadData => 'duration' in event && 'naturalSize' in event; type Props = NativeStackScreenProps; export function MediaPage({ navigation, route }: Props): React.ReactElement { @@ -39,13 +37,13 @@ export function MediaPage({ navigation, route }: Props): React.ReactElement { const isVideoPaused = !isForeground || !isScreenFocused; const [savingState, setSavingState] = useState<'none' | 'saving' | 'saved'>('none'); - const onMediaLoad = useCallback((event: OnLoadData | NativeSyntheticEvent) => { + const onMediaLoad = useCallback((event: OnLoadData | OnLoadEvent) => { if (isVideoOnLoadEvent(event)) { console.log( `Video loaded. Size: ${event.naturalSize.width}x${event.naturalSize.height} (${event.naturalSize.orientation}, ${event.duration} seconds)`, ); } else { - console.log(`Image loaded. Size: ${event.nativeEvent.source.width}x${event.nativeEvent.source.height}`); + console.log(`Image loaded. Size: ${event.nativeEvent.width}x${event.nativeEvent.height}`); } }, []); const onMediaLoadEnd = useCallback(() => { @@ -83,7 +81,7 @@ export function MediaPage({ navigation, route }: Props): React.ReactElement { return ( {type === 'photo' && ( - + )} {type === 'video' && (