Just call setSkiaFrameProcessor

mrousavy · Jul 18, 2023 · 7bf4f09 · 7bf4f09
1 parent dea9f3e
commit 7bf4f09
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 69 deletions.
diff --git a/VisionCamera.podspec b/VisionCamera.podspec
@@ -53,7 +53,6 @@ Pod::Spec.new do |s|
 
     # Skia Frame Processors
     hasSkia ? "ios/Skia Render Layer/*.{m,mm,swift}" : "",
-    hasSkia ? "ios/Skia Render Layer/SkiaFrameProcessor.h" : "",
     hasSkia ? "ios/Skia Render Layer/SkiaPreviewView.h" : "",
   ]
   # Any private headers that are not globally unique should be mentioned here.

diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock
@@ -713,7 +713,7 @@ SPEC CHECKSUMS:
   RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8
   RNVectorIcons: fcc2f6cb32f5735b586e66d14103a74ce6ad61f8
   SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
-  VisionCamera: 82c52804082e8d6cd986881ced1036b55ec1b0ba
+  VisionCamera: f3cbd24bba0067a3dece6d88ac4b8b9d44787e87
   Yoga: 65286bb6a07edce5e4fe8c90774da977ae8fc009
 
 PODFILE CHECKSUM: ab9c06b18c63e741c04349c0fd630c6d3145081c

diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx
@@ -9,6 +9,7 @@ import {
   sortFormats,
   useCameraDevices,
   useFrameProcessor,
+  useSkiaFrameProcessor,
   VideoFile,
 } from 'react-native-vision-camera';
 import { Camera, frameRateIncluded } from 'react-native-vision-camera';
@@ -218,7 +219,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
   paint.setImageFilter(imageFilter);
 
   const isIOS = Platform.OS === 'ios';
-  const frameProcessor = useFrameProcessor(
+  const frameProcessor = useSkiaFrameProcessor(
     (frame) => {
       'worklet';
       console.log(`Width: ${frame.width}`);
@@ -252,8 +253,8 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
                 video={true}
                 audio={hasMicrophonePermission}
                 enableFpsGraph={true}
-                previewType="skia"
                 orientation="portrait"
+                frameProcessor={frameProcessor}
               />
             </TapGestureHandler>
           </Reanimated.View>

diff --git a/ios/CameraView.swift b/ios/CameraView.swift
@@ -91,7 +91,13 @@ public final class CameraView: UIView {
   internal var isRecording = false
   internal var recordingSession: RecordingSession?
   #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
-    @objc public var frameProcessor: FrameProcessor?
+  @objc public var frameProcessor: FrameProcessor? {
+    didSet {
+      if let previewView = self.previewView as? SkiaPreviewView {
+        previewView.setFrameProcessor(frameProcessor)
+      }
+    }
+  }
   #endif
   // CameraView+TakePhoto
   internal var photoCaptureDelegates: [PhotoCaptureDelegate] = []
@@ -187,6 +193,9 @@ public final class CameraView: UIView {
         if previewView is SkiaPreviewView { return }
         previewView?.removeFromSuperview()
         previewView = SkiaPreviewView(frame: frame)
+        if let frameProcessor = self.frameProcessor {
+          (previewView as! SkiaPreviewView).setFrameProcessor(frameProcessor)
+        }
       #else
         invokeOnError(.system(.skiaUnavailable))
       #endif

diff --git a/ios/Skia Render Layer/SkiaFrameProcessor.mm b/ios/Skia Render Layer/SkiaFrameProcessor.mm
@@ -15,8 +15,6 @@
 #import <include/core/SkColorSpace.h>
 #import "SkImageHelpers.h"
 
-#import "SkiaMetalRenderContext.h"
-
 @implementation SkiaFrameProcessor {
   // Metal/Skia Context
   OffscreenRenderContext _offscreenContext;

diff --git a/ios/Skia Render Layer/SkiaMetalCanvasProvider.h b/ios/Skia Render Layer/SkiaMetalCanvasProvider.h
@@ -30,6 +30,8 @@ class SkiaMetalCanvasProvider: public std::enable_shared_from_this<SkiaMetalCanv
   // Update the size of the View (Layer)
   void setSize(int width, int height);
   CALayer* getLayer();
+
+  void setSkiaFrameProcessor(SkiaFrameProcessor* skiaFrameProcessor);
 
 private:
   bool _isValid = false;

diff --git a/ios/Skia Render Layer/SkiaMetalCanvasProvider.mm b/ios/Skia Render Layer/SkiaMetalCanvasProvider.mm
@@ -57,6 +57,7 @@
   NSLog(@"VisionCamera: Starting SkiaMetalCanvasProvider DisplayLink...");
   [_layerContext.displayLink start:[weakThis = weak_from_this()](double time) {
     auto thiz = weakThis.lock();
+    NSLog(@"Rendering..");
     if (thiz) {
       thiz->render();
     }
@@ -84,60 +85,62 @@
   }
 
   auto context = _layerContext.skiaContext.get();
+
+  @autoreleasepool {
+    // Create a Skia Surface from the CAMetalLayer (use to draw to the View)
+    GrMTLHandle drawableHandle;
+    auto surface = SkSurface::MakeFromCAMetalLayer(context,
+                                                   (__bridge GrMTLHandle)_layerContext.layer,
+                                                   kTopLeft_GrSurfaceOrigin,
+                                                   1,
+                                                   kBGRA_8888_SkColorType,
+                                                   nullptr,
+                                                   nullptr,
+                                                   &drawableHandle);
+    if (surface == nullptr || surface->getCanvas() == nullptr) {
+      throw std::runtime_error("Skia surface could not be created from parameters.");
+    }
 
-  // Create a Skia Surface from the CAMetalLayer (use to draw to the View)
-  GrMTLHandle drawableHandle;
-  auto surface = SkSurface::MakeFromCAMetalLayer(context,
-                                                 (__bridge GrMTLHandle)_layerContext.layer,
-                                                 kTopLeft_GrSurfaceOrigin,
-                                                 1,
-                                                 kBGRA_8888_SkColorType,
-                                                 nullptr,
-                                                 nullptr,
-                                                 &drawableHandle);
-  if (surface == nullptr || surface->getCanvas() == nullptr) {
-    throw std::runtime_error("Skia surface could not be created from parameters.");
+    auto canvas = surface->getCanvas();
+
+    // Render the latest Frame from the Frame Processor. Internally this locks and gives us a texture with the already-drawn-to Frame.
+    [_frameProcessor renderLatestFrame:^(const OffscreenRenderContext& offscreenContext) {
+      auto texture = offscreenContext.texture;
+      if (texture == nil) return;
+
+      // Calculate Center Crop (aspectRatio: cover) transform
+      auto sourceRect = SkRect::MakeXYWH(0, 0, texture.width, texture.height);
+      auto destinationRect = SkRect::MakeXYWH(0, 0, surface->width(), surface->height());
+      sourceRect = SkImageHelpers::createCenterCropRect(sourceRect, destinationRect);
+      auto offsetX = -sourceRect.left();
+      auto offsetY = -sourceRect.top();
+
+      // The Canvas is equal to the View size, where-as the Frame has a different size (e.g. 4k)
+      // We scale the Canvas to the exact dimensions of the Frame so that the user can use the Frame as a coordinate system
+      canvas->save();
+
+      auto scaleW = static_cast<double>(surface->width()) / texture.width;
+      auto scaleH = static_cast<double>(surface->height()) / texture.height;
+      auto scale = MAX(scaleW, scaleH);
+      canvas->scale(scale, scale);
+      canvas->translate(offsetX, offsetY);
+
+      // Convert the rendered MTLTexture to an SkImage
+      auto image = SkImageHelpers::convertMTLTextureToSkImage(context, texture);
+
+      // Draw the Texture (Frame) to the Canvas
+      canvas->drawImage(image, 0, 0);
+
+      // Restore the scale & transform
+      canvas->restore();
+
+      surface->flushAndSubmit();
+
+      // Pass the drawable into the Metal Command Buffer and submit it to the GPU
+      id<CAMetalDrawable> drawable = (__bridge id<CAMetalDrawable>)drawableHandle;
+      id<MTLCommandBuffer> commandBuffer([_layerContext.commandQueue commandBuffer]);
+      [commandBuffer presentDrawable:drawable];
+      [commandBuffer commit];
+    }];
   }
-
-  auto canvas = surface->getCanvas();
-
-  // Render the latest Frame from the Frame Processor. Internally this locks and gives us a texture with the already-drawn-to Frame.
-  [_frameProcessor renderLatestFrame:^(const OffscreenRenderContext& offscreenContext) {
-    auto texture = offscreenContext.texture;
-    if (texture == nil) return;
-
-    // Calculate Center Crop (aspectRatio: cover) transform
-    auto sourceRect = SkRect::MakeXYWH(0, 0, texture.width, texture.height);
-    auto destinationRect = SkRect::MakeXYWH(0, 0, surface->width(), surface->height());
-    sourceRect = SkImageHelpers::createCenterCropRect(sourceRect, destinationRect);
-    auto offsetX = -sourceRect.left();
-    auto offsetY = -sourceRect.top();
-
-    // The Canvas is equal to the View size, where-as the Frame has a different size (e.g. 4k)
-    // We scale the Canvas to the exact dimensions of the Frame so that the user can use the Frame as a coordinate system
-    canvas->save();
-
-    auto scaleW = static_cast<double>(surface->width()) / texture.width;
-    auto scaleH = static_cast<double>(surface->height()) / texture.height;
-    auto scale = MAX(scaleW, scaleH);
-    canvas->scale(scale, scale);
-    canvas->translate(offsetX, offsetY);
-
-    // Convert the rendered MTLTexture to an SkImage
-    auto image = SkImageHelpers::convertMTLTextureToSkImage(context, texture);
-
-    // Draw the Texture (Frame) to the Canvas
-    canvas->drawImage(image, 0, 0);
-
-    // Restore the scale & transform
-    canvas->restore();
-
-    surface->flushAndSubmit();
-
-    // Pass the drawable into the Metal Command Buffer and submit it to the GPU
-    id<CAMetalDrawable> drawable = (__bridge id<CAMetalDrawable>)drawableHandle;
-    id<MTLCommandBuffer> commandBuffer([_layerContext.commandQueue commandBuffer]);
-    [commandBuffer presentDrawable:drawable];
-    [commandBuffer commit];
-  }];
 }
diff --git a/ios/Skia Render Layer/SkiaMetalRenderContext.h b/ios/Skia Render Layer/SkiaMetalRenderContext.h
@@ -6,8 +6,7 @@
 //  Copyright © 2022 mrousavy. All rights reserved.
 //
 
-#ifndef SkiaMetalRenderContext_h
-#define SkiaMetalRenderContext_h
+#pragma once
 
 #import <MetalKit/MetalKit.h>
 #import <QuartzCore/CAMetalLayer.h>
@@ -38,5 +37,3 @@ struct LayerRenderContext: public RenderContext {
   CAMetalLayer* layer;
   VisionDisplayLink* displayLink;
 };
-
-#endif /* SkiaMetalRenderContext_h */
diff --git a/ios/Skia Render Layer/SkiaPreviewView.h b/ios/Skia Render Layer/SkiaPreviewView.h
@@ -11,10 +11,10 @@
 #import <UIKit/UIKit.h>
 #import <AVFoundation/AVFoundation.h>
 #import "PreviewView.h"
-#import "SkiaFrameProcessor.h"
+#import "FrameProcessor.h"
 
 @interface SkiaPreviewView: PreviewView
 
-- (instancetype _Nonnull ) initWithRenderProxy:(SkiaRenderProxy* _Nonnull)renderProxy;
+- (void)setFrameProcessor:(FrameProcessor*)frameProcessor;
 
 @end
diff --git a/src/hooks/useFrameProcessor.ts b/src/hooks/useFrameProcessor.ts
@@ -1,5 +1,5 @@
 import { DependencyList, useMemo } from 'react';
-import type { Frame, FrameInternal } from '../Frame';
+import type { DrawableFrame, Frame, FrameInternal } from '../Frame';
 import { FrameProcessor } from '../CameraProps';
 // Install RN Worklets by importing it
 import 'react-native-worklets/src';
@@ -64,7 +64,7 @@ export function useFrameProcessor(frameProcessor: (frame: Frame) => void, depend
  * }, [])
  * ```
  */
-export function useSkiaFrameProcessor(frameProcessor: (frame: Frame) => void, dependencies: DependencyList): FrameProcessor {
+export function useSkiaFrameProcessor(frameProcessor: (frame: DrawableFrame) => void, dependencies: DependencyList): FrameProcessor {
   // eslint-disable-next-line react-hooks/exhaustive-deps
   const fp = useFrameProcessor(frameProcessor, dependencies);
   fp.type = 'skia-frame-processor';