diff --git a/package/ios/React/CameraViewManager.swift b/package/ios/React/CameraViewManager.swift index 4cd7c1bdf9..a83de59ee2 100644 --- a/package/ios/React/CameraViewManager.swift +++ b/package/ios/React/CameraViewManager.swift @@ -94,14 +94,14 @@ final class CameraViewManager: RCTViewManager { promise.reject(error: .parameter(.invalid(unionName: "focusOptions", receivedValue: focusOptions.description))) return } - guard let x = pointDictionary["x"] as? NSNumber, let y = pointDictionary["y"] as? NSNumber else { - promise.reject(error: .parameter(.invalid(unionName: "focusOptions.point", receivedValue: pointDictionary.description))) - return - } guard let coordinateSystem = try? CoordinateSystem(jsValue: coordinateSystemString) else { promise.reject(error: .parameter(.invalid(unionName: "focusOptions.coordinateSystem", receivedValue: coordinateSystemString))) return } + guard let x = pointDictionary["x"] as? NSNumber, let y = pointDictionary["y"] as? NSNumber else { + promise.reject(error: .parameter(.invalid(unionName: "focusOptions.point", receivedValue: pointDictionary.description))) + return + } let component = getCameraView(withTag: node) let point = CGPoint(x: x.doubleValue, y: y.doubleValue) diff --git a/package/src/skia/useSkiaFrameProcessor.ts b/package/src/skia/useSkiaFrameProcessor.ts index 8c8a92d2f2..871c004c8f 100644 --- a/package/src/skia/useSkiaFrameProcessor.ts +++ b/package/src/skia/useSkiaFrameProcessor.ts @@ -9,6 +9,7 @@ import { SkiaProxy } from '../dependencies/SkiaProxy' import { withFrameRefCounting } from '../frame-processors/withFrameRefCounting' import { VisionCameraProxy } from '../frame-processors/VisionCameraProxy' import type { Orientation } from '../types/Orientation' +import type { CameraMatrix } from '../types/CameraMatrix' /** * Represents a Camera Frame that can be directly drawn to using Skia. @@ -141,6 +142,28 @@ function getSurfaceSize(frame: Frame): Size { } } +interface SkiaFrameProcessorState { + surfaceHolder: ISharedValue + offscreenTextures: ISharedValue + previewOrientation: ISharedValue + cameraMatrix: ISharedValue +} + +export function createSkiaFrameProcessorState(): SkiaFrameProcessorState { + const Worklets = WorkletsProxy.Worklets + return { + surfaceHolder: Worklets.createSharedValue({}), + offscreenTextures: Worklets.createSharedValue([]), + previewOrientation: Worklets.createSharedValue('portrait'), + cameraMatrix: Worklets.createSharedValue({ + width: 0, + height: 0, + orientation: 'portrait', + isMirrored: false, + }), + } +} + /** * Create a new Frame Processor function which you can pass to the ``. * (See ["Frame Processors"](https://react-native-vision-camera.com/docs/guides/frame-processors)) @@ -152,8 +175,7 @@ function getSurfaceSize(frame: Frame): Size { * @worklet * @example * ```ts - * const surfaceHolder = Worklets.createSharedValue({}) - * const offscreenTextures = Worklets.createSharedValue([]) + * const state = createSkiaFrameProcessorState() * const frameProcessor = createSkiaFrameProcessor((frame) => { * 'worklet' * const faces = scanFaces(frame) @@ -163,17 +185,16 @@ function getSurfaceSize(frame: Frame): Size { * const rect = Skia.XYWHRect(face.x, face.y, face.width, face.height) * frame.drawRect(rect) * } - * }, surfaceHolder, offscreenTextures) + * }, state) * ``` */ export function createSkiaFrameProcessor( frameProcessor: (frame: DrawableFrame) => void, - surfaceHolder: ISharedValue, - offscreenTextures: ISharedValue, - previewOrientation: ISharedValue, + state: SkiaFrameProcessorState, ): DrawableFrameProcessor { const Skia = SkiaProxy.Skia const Worklets = WorkletsProxy.Worklets + const { cameraMatrix, offscreenTextures, previewOrientation, surfaceHolder } = state const getSkiaSurface = (frame: Frame): SkSurface => { 'worklet' @@ -251,6 +272,26 @@ export function createSkiaFrameProcessor( return (frame as FrameInternal).withBaseClass(canvasProxy) } + const updateCameraMatrix = (frame: Frame): void => { + 'worklet' + + const currentMatrix = cameraMatrix.value + if ( + currentMatrix.width !== frame.width || + currentMatrix.height !== frame.height || + currentMatrix.isMirrored !== frame.isMirrored || + currentMatrix.orientation !== frame.orientation + ) { + // Update Matrix + cameraMatrix.value = { + width: frame.width, + height: frame.height, + isMirrored: frame.isMirrored, + orientation: frame.orientation, + } + } + } + return { frameProcessor: withFrameRefCounting((frame) => { 'worklet' @@ -293,10 +334,14 @@ export function createSkiaFrameProcessor( if (texture == null) break texture.dispose() } + + // 10. After rendering, update the Camera Matrix used for view -> camera conversions + updateCameraMatrix(frame) }), type: 'drawable-skia', offscreenTextures: offscreenTextures, previewOrientation: previewOrientation, + cameraMatrix: cameraMatrix, } } @@ -332,9 +377,8 @@ export function useSkiaFrameProcessor( frameProcessor: (frame: DrawableFrame) => void, dependencies: DependencyList, ): DrawableFrameProcessor { - const surface = WorkletsProxy.useSharedValue({}) - const offscreenTextures = WorkletsProxy.useSharedValue([]) - const previewOrientation = WorkletsProxy.useSharedValue('portrait') + // Creates SharedValues that will be kept in state for the Frame Processor + const state = useMemo(() => createSkiaFrameProcessorState(), []) useEffect(() => { return () => { @@ -343,20 +387,20 @@ export function useSkiaFrameProcessor( // if it is currently executing - so we avoid race conditions here. VisionCameraProxy.workletContext?.runAsync(() => { 'worklet' - const surfaces = Object.values(surface.value).map((v) => v.surface) - surface.value = {} + const surfaces = Object.values(state.surfaceHolder.value).map((v) => v.surface) + state.surfaceHolder.value = {} surfaces.forEach((s) => s.dispose()) - while (offscreenTextures.value.length > 0) { - const texture = offscreenTextures.value.shift() + while (state.offscreenTextures.value.length > 0) { + const texture = state.offscreenTextures.value.shift() if (texture == null) break texture.dispose() } }) } - }, [offscreenTextures, surface]) + }, [state.surfaceHolder, state.offscreenTextures]) return useMemo( - () => createSkiaFrameProcessor(frameProcessor, surface, offscreenTextures, previewOrientation), + () => createSkiaFrameProcessor(frameProcessor, state), // eslint-disable-next-line react-hooks/exhaustive-deps dependencies, ) diff --git a/package/src/types/CameraMatrix.ts b/package/src/types/CameraMatrix.ts new file mode 100644 index 0000000000..3dee3f22a0 --- /dev/null +++ b/package/src/types/CameraMatrix.ts @@ -0,0 +1,8 @@ +import type { Orientation } from './Orientation' + +export interface CameraMatrix { + orientation: Orientation + isMirrored: boolean + width: number + height: number +} diff --git a/package/src/types/CameraProps.ts b/package/src/types/CameraProps.ts index 8056d35c95..8ee2d88807 100644 --- a/package/src/types/CameraProps.ts +++ b/package/src/types/CameraProps.ts @@ -7,6 +7,7 @@ import type { ISharedValue } from 'react-native-worklets-core' import type { SkImage } from '@shopify/react-native-skia' import type { OutputOrientation } from './OutputOrientation' import type { Orientation } from './Orientation' +import type { CameraMatrix } from './CameraMatrix' export interface ReadonlyFrameProcessor { frameProcessor: (frame: Frame) => void @@ -17,6 +18,7 @@ export interface DrawableFrameProcessor { type: 'drawable-skia' offscreenTextures: ISharedValue previewOrientation: ISharedValue + cameraMatrix: ISharedValue } export interface OnShutterEvent {