feat: Create cameraMatrix

package/ios/React/CameraViewManager.swift

@@ -94,14 +94,14 @@ final class CameraViewManager: RCTViewManager {
             promise.reject(error: .parameter(.invalid(unionName: "focusOptions", receivedValue: focusOptions.description)))
             return
     }
-    guard let x = pointDictionary["x"] as? NSNumber, let y = pointDictionary["y"] as? NSNumber else {
-      promise.reject(error: .parameter(.invalid(unionName: "focusOptions.point", receivedValue: pointDictionary.description)))
-      return
-    }
     guard let coordinateSystem = try? CoordinateSystem(jsValue: coordinateSystemString) else {
       promise.reject(error: .parameter(.invalid(unionName: "focusOptions.coordinateSystem", receivedValue: coordinateSystemString)))
       return
     }
+    guard let x = pointDictionary["x"] as? NSNumber, let y = pointDictionary["y"] as? NSNumber else {
+      promise.reject(error: .parameter(.invalid(unionName: "focusOptions.point", receivedValue: pointDictionary.description)))
+      return
+    }
     let component = getCameraView(withTag: node)
     let point = CGPoint(x: x.doubleValue, y: y.doubleValue)
 

package/src/skia/useSkiaFrameProcessor.ts

@@ -9,6 +9,7 @@ import { SkiaProxy } from '../dependencies/SkiaProxy'
 import { withFrameRefCounting } from '../frame-processors/withFrameRefCounting'
 import { VisionCameraProxy } from '../frame-processors/VisionCameraProxy'
 import type { Orientation } from '../types/Orientation'
+import type { CameraMatrix } from '../types/CameraMatrix'
 
 /**
  * Represents a Camera Frame that can be directly drawn to using Skia.
@@ -141,6 +142,28 @@ function getSurfaceSize(frame: Frame): Size {
   }
 }
 
+interface SkiaFrameProcessorState {
+  surfaceHolder: ISharedValue<SurfaceCache>
+  offscreenTextures: ISharedValue<SkImage[]>
+  previewOrientation: ISharedValue<Orientation>
+  cameraMatrix: ISharedValue<CameraMatrix>
+}
+
+export function createSkiaFrameProcessorState(): SkiaFrameProcessorState {
+  const Worklets = WorkletsProxy.Worklets
+  return {
+    surfaceHolder: Worklets.createSharedValue({}),
+    offscreenTextures: Worklets.createSharedValue([]),
+    previewOrientation: Worklets.createSharedValue('portrait'),
+    cameraMatrix: Worklets.createSharedValue({
+      width: 0,
+      height: 0,
+      orientation: 'portrait',
+      isMirrored: false,
+    }),
+  }
+}
+
 /**
  * Create a new Frame Processor function which you can pass to the `<Camera>`.
  * (See ["Frame Processors"](https://react-native-vision-camera.com/docs/guides/frame-processors))
@@ -152,8 +175,7 @@ function getSurfaceSize(frame: Frame): Size {
  * @worklet
  * @example
  * ```ts
- * const surfaceHolder = Worklets.createSharedValue<SurfaceCache>({})
- * const offscreenTextures = Worklets.createSharedValue<SkImage[]>([])
+ * const state = createSkiaFrameProcessorState()
  * const frameProcessor = createSkiaFrameProcessor((frame) => {
  *   'worklet'
  *   const faces = scanFaces(frame)
@@ -163,17 +185,16 @@ function getSurfaceSize(frame: Frame): Size {
  *     const rect = Skia.XYWHRect(face.x, face.y, face.width, face.height)
  *     frame.drawRect(rect)
  *   }
- * }, surfaceHolder, offscreenTextures)
+ * }, state)
  * ```
  */
 export function createSkiaFrameProcessor(
   frameProcessor: (frame: DrawableFrame) => void,
-  surfaceHolder: ISharedValue<SurfaceCache>,
-  offscreenTextures: ISharedValue<SkImage[]>,
-  previewOrientation: ISharedValue<Orientation>,
+  state: SkiaFrameProcessorState,
 ): DrawableFrameProcessor {
   const Skia = SkiaProxy.Skia
   const Worklets = WorkletsProxy.Worklets
+  const { cameraMatrix, offscreenTextures, previewOrientation, surfaceHolder } = state
 
   const getSkiaSurface = (frame: Frame): SkSurface => {
     'worklet'
@@ -251,6 +272,26 @@ export function createSkiaFrameProcessor(
     return (frame as FrameInternal).withBaseClass(canvasProxy)
   }
 
+  const updateCameraMatrix = (frame: Frame): void => {
+    'worklet'
+
+    const currentMatrix = cameraMatrix.value
+    if (
+      currentMatrix.width !== frame.width ||
+      currentMatrix.height !== frame.height ||
+      currentMatrix.isMirrored !== frame.isMirrored ||
+      currentMatrix.orientation !== frame.orientation
+    ) {
+      // Update Matrix
+      cameraMatrix.value = {
+        width: frame.width,
+        height: frame.height,
+        isMirrored: frame.isMirrored,
+        orientation: frame.orientation,
+      }
+    }
+  }
+
   return {
     frameProcessor: withFrameRefCounting((frame) => {
       'worklet'
@@ -293,10 +334,14 @@ export function createSkiaFrameProcessor(
         if (texture == null) break
         texture.dispose()
       }
+
+      // 10. After rendering, update the Camera Matrix used for view -> camera conversions
+      updateCameraMatrix(frame)
     }),
     type: 'drawable-skia',
     offscreenTextures: offscreenTextures,
     previewOrientation: previewOrientation,
+    cameraMatrix: cameraMatrix,
   }
 }
 
@@ -332,9 +377,8 @@ export function useSkiaFrameProcessor(
   frameProcessor: (frame: DrawableFrame) => void,
   dependencies: DependencyList,
 ): DrawableFrameProcessor {
-  const surface = WorkletsProxy.useSharedValue<SurfaceCache>({})
-  const offscreenTextures = WorkletsProxy.useSharedValue<SkImage[]>([])
-  const previewOrientation = WorkletsProxy.useSharedValue<Orientation>('portrait')
+  // Creates SharedValues that will be kept in state for the Frame Processor
+  const state = useMemo(() => createSkiaFrameProcessorState(), [])
 
   useEffect(() => {
     return () => {
@@ -343,20 +387,20 @@ export function useSkiaFrameProcessor(
       // if it is currently executing - so we avoid race conditions here.
       VisionCameraProxy.workletContext?.runAsync(() => {
         'worklet'
-        const surfaces = Object.values(surface.value).map((v) => v.surface)
-        surface.value = {}
+        const surfaces = Object.values(state.surfaceHolder.value).map((v) => v.surface)
+        state.surfaceHolder.value = {}
         surfaces.forEach((s) => s.dispose())
-        while (offscreenTextures.value.length > 0) {
-          const texture = offscreenTextures.value.shift()
+        while (state.offscreenTextures.value.length > 0) {
+          const texture = state.offscreenTextures.value.shift()
           if (texture == null) break
           texture.dispose()
         }
       })
     }
-  }, [offscreenTextures, surface])
+  }, [state.surfaceHolder, state.offscreenTextures])
 
   return useMemo(
-    () => createSkiaFrameProcessor(frameProcessor, surface, offscreenTextures, previewOrientation),
+    () => createSkiaFrameProcessor(frameProcessor, state),
     // eslint-disable-next-line react-hooks/exhaustive-deps
     dependencies,
   )

package/src/types/CameraMatrix.ts

@@ -0,0 +1,8 @@
+import type { Orientation } from './Orientation'
+
+export interface CameraMatrix {
+  orientation: Orientation
+  isMirrored: boolean
+  width: number
+  height: number
+}

package/src/types/CameraProps.ts

@@ -7,6 +7,7 @@ import type { ISharedValue } from 'react-native-worklets-core'
 import type { SkImage } from '@shopify/react-native-skia'
 import type { OutputOrientation } from './OutputOrientation'
 import type { Orientation } from './Orientation'
+import type { CameraMatrix } from './CameraMatrix'
 
 export interface ReadonlyFrameProcessor {
   frameProcessor: (frame: Frame) => void
@@ -17,6 +18,7 @@ export interface DrawableFrameProcessor {
   type: 'drawable-skia'
   offscreenTextures: ISharedValue<SkImage[]>
   previewOrientation: ISharedValue<Orientation>
+  cameraMatrix: ISharedValue<CameraMatrix>
 }
 
 export interface OnShutterEvent {