feat: Android Camera2 rewrite

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -27,6 +27,11 @@ body:
       label: Relevant log output
       description: Please copy and paste any relevant log output (Xcode Logs/Android Studio Logcat). This will be automatically formatted into code, so no need for backticks.
       render: shell
+  - type: textarea
+    attributes:
+      label: Camera Device
+      description: Please paste the JSON Camera `device` that was used here. (`console.log(JSON.stringify(device, null, 2))`) This will be automatically formatted into code, so no need for backticks.
+      render: shell
   - type: input
     attributes:
       label: Device

.github/workflows/build-android.yml

@@ -127,8 +127,8 @@ jobs:
         run: yarn install --frozen-lockfile
       - name: Install node_modules for example/
         run: yarn install --frozen-lockfile --cwd example
-      - name: Remove react-native-worklets
-        run: yarn remove react-native-worklets --cwd example
+      - name: Remove react-native-worklets-core
+        run: yarn remove react-native-worklets-core --cwd example
 
       - name: Restore Gradle cache
         uses: actions/cache@v2

.github/workflows/build-ios.yml

@@ -163,8 +163,8 @@ jobs:
             ${{ runner.os }}-yarn-
       - name: Install node_modules for example/
         run: yarn install --frozen-lockfile --cwd ..
-      - name: Remove react-native-worklets
-        run: yarn remove react-native-worklets --cwd ..
+      - name: Remove react-native-worklets-core
+        run: yarn remove react-native-worklets-core --cwd ..
 
       - name: Restore buildcache
         uses: mikehardy/buildcache-action@v1

README.md

@@ -42,7 +42,7 @@
 
 ### Features
 
-* Photo, Video and Snapshot capture
+* Photo and Video capture
 * Customizable devices and multi-cameras (smoothly zoom out to "fish-eye" camera)
 * Customizable FPS
 * [Frame Processors](https://react-native-vision-camera.com/docs/guides/frame-processors) (JS worklets to run QR-Code scanning, facial recognition, AI object detection, realtime video chats, ...)

VisionCamera.podspec

@@ -22,9 +22,9 @@ if defined?($VCDisableSkia)
 end
 
 Pod::UI.puts("[VisionCamera] node modules #{Dir.exist?(nodeModules) ? "found at #{nodeModules}" : "not found!"}")
-workletsPath = File.join(nodeModules, "react-native-worklets")
+workletsPath = File.join(nodeModules, "react-native-worklets-core")
 hasWorklets = File.exist?(workletsPath) && !forceDisableFrameProcessors
-Pod::UI.puts("[VisionCamera] react-native-worklets #{hasWorklets ? "found" : "not found"}, Frame Processors #{hasWorklets ? "enabled" : "disabled"}!")
+Pod::UI.puts("[VisionCamera] react-native-worklets-core #{hasWorklets ? "found" : "not found"}, Frame Processors #{hasWorklets ? "enabled" : "disabled"}!")
 
 skiaPath = File.join(nodeModules, "@shopify", "react-native-skia")
 hasSkia = hasWorklets && File.exist?(skiaPath) && !forceDisableSkia
@@ -87,7 +87,7 @@ Pod::Spec.new do |s|
   s.dependency "React-callinvoker"
 
   if hasWorklets
-    s.dependency "react-native-worklets"
+    s.dependency "react-native-worklets-core"
     if hasSkia
       s.dependency "react-native-skia"
     end

android/CMakeLists.txt

@@ -6,6 +6,7 @@ set(PACKAGE_NAME "VisionCamera")
 set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
 set(CMAKE_VERBOSE_MAKEFILE ON)
 set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSK_GL -DSK_GANESH -DSK_BUILD_FOR_ANDROID")
 
 # Folly
 include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake")
@@ -14,10 +15,9 @@ add_compile_options(${folly_FLAGS})
 # Third party libraries (Prefabs)
 find_package(ReactAndroid REQUIRED CONFIG)
 find_package(fbjni REQUIRED CONFIG)
-find_package(react-native-worklets REQUIRED CONFIG)
+find_package(react-native-worklets-core REQUIRED CONFIG)
 find_library(LOG_LIB log)
 
-
 set(RNSKIA_PATH ${NODE_MODULES_DIR}/@shopify/react-native-skia)
 if(EXISTS ${RNSKIA_PATH})
     find_package(shopify_react-native-skia REQUIRED CONFIG)
@@ -27,6 +27,14 @@ else()
     message("VisionCamera: Skia integration disabled!")
 ENDIF()
 
+set (SKIA_LIBS_PATH "${RNSKIA_PATH}/libs/android/${ANDROID_ABI}")
+add_library(skia STATIC IMPORTED)
+set_property(TARGET skia PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskia.a")
+add_library(svg STATIC IMPORTED)
+set_property(TARGET svg PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libsvg.a")
+add_library(skshaper STATIC IMPORTED)
+set_property(TARGET skshaper PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskshaper.a")
+
 # Add react-native-vision-camera sources
 add_library(
         ${PACKAGE_NAME}
@@ -37,6 +45,7 @@ add_library(
         src/main/cpp/JSIJNIConversion.cpp
         src/main/cpp/VisionCamera.cpp
         src/main/cpp/VisionCameraProxy.cpp
+        src/main/cpp/skia/SkiaRenderer.cpp
         src/main/cpp/java-bindings/JFrame.cpp
         src/main/cpp/java-bindings/JFrameProcessor.cpp
         src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp
@@ -62,6 +71,16 @@ target_include_directories(
         #  just one directory. HOWEVER, skia itself uses relative paths in
         #  their include statements, and so we have to include the path to skia)
         "${RNSKIA_PATH}/cpp/skia"
+
+        "${RNSKIA_PATH}/cpp/skia/include/config/"
+        "${RNSKIA_PATH}/cpp/skia/include/core/"
+        "${RNSKIA_PATH}/cpp/skia/include/effects/"
+        "${RNSKIA_PATH}/cpp/skia/include/utils/"
+        "${RNSKIA_PATH}/cpp/skia/include/pathops/"
+        "${RNSKIA_PATH}/cpp/skia/modules/"
+        # "${RNSKIA_PATH}/cpp/skia/modules/skparagraph/include/"
+        "${RNSKIA_PATH}/cpp/skia/include/"
+        "${RNSKIA_PATH}/cpp/skia"
 )
 
 # Link everything together
@@ -73,6 +92,12 @@ target_link_libraries(
         ReactAndroid::reactnativejni        # <-- RN: React Native JNI bindings
         ReactAndroid::folly_runtime         # <-- RN: For casting JSI <> Java objects
         fbjni::fbjni                        # <-- fbjni
-        react-native-worklets::rnworklets   # <-- RN Worklets
+        react-native-worklets-core::rnworklets   # <-- RN Worklets
+        GLESv2                              # <-- Optional: OpenGL (for Skia)
+        EGL                                 # <-- Optional: OpenGL (EGL) (for Skia)
         ${SKIA_PACKAGE}                     # <-- Optional: RN Skia
+        jnigraphics
+        skia
+        svg
+        skshaper
 )

android/build.gradle

@@ -142,22 +142,13 @@ dependencies {
   //noinspection GradleDynamicVersion
   implementation 'com.facebook.react:react-android:+'
 
-  implementation 'androidx.core:core-ktx:1.3.2'
+  implementation "androidx.core:core-ktx:1.3.2"
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
   implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.5.2"
   implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2"
+  implementation "androidx.exifinterface:exifinterface:1.3.6"
 
-  implementation "androidx.camera:camera-core:1.1.0"
-  implementation "androidx.camera:camera-camera2:1.1.0"
-  implementation "androidx.camera:camera-lifecycle:1.1.0"
-  implementation "androidx.camera:camera-video:1.1.0"
-
-  implementation "androidx.camera:camera-view:1.1.0"
-  implementation "androidx.camera:camera-extensions:1.1.0"
-
-  implementation "androidx.exifinterface:exifinterface:1.3.3"
-
-  implementation project(":react-native-worklets")
+  implementation project(":react-native-worklets-core")
   implementation project(":shopify_react-native-skia")
 }
 

android/src/main/cpp/FrameHostObject.cpp

@@ -8,7 +8,7 @@
 #include <fbjni/fbjni.h>
 #include <jni.h>
 
-#include <react-native-worklets/WKTJsiHostObject.h>
+#include <react-native-worklets-core/WKTJsiHostObject.h>
 #include "JSITypedArray.h"
 
 #include <vector>
@@ -18,7 +18,7 @@ namespace vision {
 
 using namespace facebook;
 
-FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame): frame(make_global(frame)), _refCount(0) { }
+FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame): frame(make_global(frame)) { }
 
 FrameHostObject::~FrameHostObject() {
   // Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread
@@ -37,6 +37,7 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("pixelFormat")));
   // Conversion
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer")));
@@ -94,8 +95,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   if (name == "incrementRefCount") {
     auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
       // Increment retain count by one.
-      std::lock_guard lock(this->_refCountMutex);
-      this->_refCount++;
+      this->frame->incrementRefCount();
       return jsi::Value::undefined();
     };
     return jsi::Function::createFromHostFunction(runtime,
@@ -106,12 +106,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
 
   if (name == "decrementRefCount") {
     auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
-      // Decrement retain count by one. If the retain count is zero, we close the Frame.
-      std::lock_guard lock(this->_refCountMutex);
-      this->_refCount--;
-      if (_refCount < 1) {
-        this->frame->close();
-      }
+      // Decrement retain count by one. If the retain count is zero, the Frame gets closed.
+      this->frame->decrementRefCount();
       return jsi::Value::undefined();
     };
     return jsi::Function::createFromHostFunction(runtime,
@@ -136,6 +132,10 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
     auto string = this->frame->getOrientation();
     return jsi::String::createFromUtf8(runtime, string->toStdString());
   }
+  if (name == "pixelFormat") {
+    auto string = this->frame->getPixelFormat();
+    return jsi::String::createFromUtf8(runtime, string->toStdString());
+  }
   if (name == "timestamp") {
     return jsi::Value(static_cast<double>(this->frame->getTimestamp()));
   }

android/src/main/cpp/FrameHostObject.h

@@ -9,7 +9,6 @@
 #include <fbjni/fbjni.h>
 #include <vector>
 #include <string>
-#include <mutex>
 
 #include "java-bindings/JFrame.h"
 
@@ -31,9 +30,6 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
 
  private:
   static auto constexpr TAG = "VisionCamera";
-
-  size_t _refCount;
-  std::mutex _refCountMutex;
 };
 
 } // namespace vision

android/src/main/cpp/VisionCamera.cpp

@@ -4,12 +4,14 @@
 #include "java-bindings/JFrameProcessor.h"
 #include "java-bindings/JVisionCameraProxy.h"
 #include "VisionCameraProxy.h"
+#include "skia/SkiaRenderer.h"
 
 JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
   return facebook::jni::initialize(vm, [] {
     vision::VisionCameraInstaller::registerNatives();
     vision::JFrameProcessor::registerNatives();
     vision::JVisionCameraProxy::registerNatives();
     vision::JVisionCameraScheduler::registerNatives();
+    vision::SkiaRenderer::registerNatives();
   });
 }

android/src/main/cpp/java-bindings/JFrame.cpp

@@ -42,6 +42,11 @@ local_ref<JString> JFrame::getOrientation() const {
   return getOrientationMethod(self());
 }
 
+local_ref<JString> JFrame::getPixelFormat() const {
+  static const auto getPixelFormatMethod = getClass()->getMethod<JString()>("getPixelFormat");
+  return getPixelFormatMethod(self());
+}
+
 int JFrame::getPlanesCount() const {
   static const auto getPlanesCountMethod = getClass()->getMethod<jint()>("getPlanesCount");
   return getPlanesCountMethod(self());
@@ -57,6 +62,16 @@ local_ref<JArrayByte> JFrame::toByteArray() const {
   return toByteArrayMethod(self());
 }
 
+void JFrame::incrementRefCount() {
+  static const auto incrementRefCountMethod = getClass()->getMethod<void()>("incrementRefCount");
+  incrementRefCountMethod(self());
+}
+
+void JFrame::decrementRefCount() {
+  static const auto decrementRefCountMethod = getClass()->getMethod<void()>("decrementRefCount");
+  decrementRefCountMethod(self());
+}
+
 void JFrame::close() {
   static const auto closeMethod = getClass()->getMethod<void()>("close");
   closeMethod(self());

android/src/main/cpp/java-bindings/JFrame.h

@@ -24,7 +24,10 @@ struct JFrame : public JavaClass<JFrame> {
   int getBytesPerRow() const;
   jlong getTimestamp() const;
   local_ref<JString> getOrientation() const;
+  local_ref<JString> getPixelFormat() const;
   local_ref<JArrayByte> toByteArray() const;
+  void incrementRefCount();
+  void decrementRefCount();
   void close();
 };
 

android/src/main/cpp/java-bindings/JFrameProcessor.h

@@ -9,8 +9,8 @@
 #include <jni.h>
 #include <fbjni/fbjni.h>
 
-#include <react-native-worklets/WKTJsiWorklet.h>
-#include <react-native-worklets/WKTJsiHostObject.h>
+#include <react-native-worklets-core/WKTJsiWorklet.h>
+#include <react-native-worklets-core/WKTJsiHostObject.h>
 
 #include "JFrame.h"
 #include "FrameHostObject.h"

android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp

@@ -11,8 +11,8 @@
 #include <jsi/jsi.h>
 #include <react/jni/ReadableNativeMap.h>
 
-#include <react-native-worklets/WKTJsiWorklet.h>
-#include <react-native-worklets/WKTJsiWorkletContext.h>
+#include <react-native-worklets-core/WKTJsiWorklet.h>
+#include <react-native-worklets-core/WKTJsiWorkletContext.h>
 
 #include "FrameProcessorPluginHostObject.h"
 

android/src/main/cpp/java-bindings/JVisionCameraProxy.h

@@ -6,7 +6,7 @@
 
 #include <fbjni/fbjni.h>
 #include <jsi/jsi.h>
-#include <react-native-worklets/WKTJsiWorkletContext.h>
+#include <react-native-worklets-core/WKTJsiWorkletContext.h>
 #include <react/jni/ReadableNativeMap.h>
 
 #include "JFrameProcessorPlugin.h"

android/src/main/cpp/skia/OpenGLError.h

@@ -0,0 +1,26 @@
+//
+// Created by Marc Rousavy on 09.08.23.
+//
+
+#pragma once
+
+#include <string>
+#include <stdexcept>
+#include <GLES2/gl2.h>
+
+namespace vision {
+
+inline std::string getEglErrorIfAny() {
+  EGLint error = glGetError();
+  if (error != GL_NO_ERROR) return " Error: " + std::to_string(error);
+  error = eglGetError();
+  if (error != EGL_SUCCESS) return " Error: " + std::to_string(error);
+  return "";
+}
+
+class OpenGLError: public std::runtime_error {
+ public:
+  explicit OpenGLError(const std::string&& message): std::runtime_error(message + getEglErrorIfAny()) {}
+};
+
+} // namespace vision