First Android rewrite

android/CMakeLists.txt

@@ -32,15 +32,17 @@ add_library(
         ${PACKAGE_NAME}
         SHARED
         ../cpp/JSITypedArray.cpp
-        src/main/cpp/VisionCamera.cpp
-        src/main/cpp/JSIJNIConversion.cpp
         src/main/cpp/FrameHostObject.cpp
-        src/main/cpp/FrameProcessorRuntimeManager.cpp
-        src/main/cpp/CameraView.cpp
-        src/main/cpp/VisionCameraScheduler.cpp
+        src/main/cpp/FrameProcessorPluginHostObject.cpp
+        src/main/cpp/JSIJNIConversion.cpp
+        src/main/cpp/VisionCamera.cpp
+        src/main/cpp/VisionCameraProxy.cpp
+        src/main/cpp/java-bindings/JCameraView.cpp
+        src/main/cpp/java-bindings/JFrame.cpp
+        src/main/cpp/java-bindings/JFrameProcessor.cpp
         src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp
-        src/main/cpp/java-bindings/JImageProxy.cpp
         src/main/cpp/java-bindings/JHashMap.cpp
+        src/main/cpp/java-bindings/JVisionCameraScheduler.cpp
 )
 
 # Header Search Paths (includes)

android/src/main/cpp/FrameHostObject.cpp

@@ -18,7 +18,7 @@ namespace vision {
 
 using namespace facebook;
 
-FrameHostObject::FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image): frame(make_global(image)), _refCount(0) { }
+FrameHostObject::FrameHostObject(jni::alias_ref<JFrame::javaobject> frame): frame(make_global(frame)), _refCount(0) { }
 
 FrameHostObject::~FrameHostObject() {
   // Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread

android/src/main/cpp/FrameHostObject.h

@@ -11,23 +11,23 @@
 #include <string>
 #include <mutex>
 
-#include "java-bindings/JImageProxy.h"
+#include "java-bindings/JFrame.h"
 
 namespace vision {
 
 using namespace facebook;
 
 class JSI_EXPORT FrameHostObject : public jsi::HostObject {
  public:
-  explicit FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image);
+  explicit FrameHostObject(jni::alias_ref<JFrame::javaobject> frame);
   ~FrameHostObject();
 
  public:
   jsi::Value get(jsi::Runtime &, const jsi::PropNameID &name) override;
   std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
 
  public:
-  jni::global_ref<JImageProxy> frame;
+  jni::global_ref<JFrame> frame;
 
  private:
   static auto constexpr TAG = "VisionCamera";

android/src/main/cpp/FrameProcessorPluginHostObject.cpp

@@ -0,0 +1,52 @@
+//
+// Created by Marc Rousavy on 21.07.23.
+//
+
+#include "FrameProcessorPluginHostObject.h"
+#include <vector>
+#include "FrameHostObject.h"
+#include "JSIJNIConversion.h"
+
+namespace vision {
+
+using namespace facebook;
+
+std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime &runtime) {
+  std::vector<jsi::PropNameID> result;
+  result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
+  return result;
+}
+
+jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime &runtime, const jsi::PropNameID &propName) {
+  auto name = propName.utf8(runtime);
+
+  if (name == "call") {
+    return jsi::Function::createFromHostFunction(runtime,
+                                                 jsi::PropNameID::forUtf8(runtime, "call"),
+                                                 2,
+                                                 [=](jsi::Runtime &runtime,
+                                                     const jsi::Value &thisValue,
+                                                     const jsi::Value *arguments,
+                                                     size_t count) -> jsi::Value {
+      // Frame is first argument
+      auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
+      auto frame = frameHostObject->frame;
+
+      // Options are second argument (possibly undefined)
+      jobject options = nullptr;
+      if (count > 1) {
+        options = JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[1]);
+      }
+
+      // Call actual plugin
+      auto result = _plugin->callback(frame, options);
+
+      // Convert result value to jsi::Value (possibly undefined)
+      return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
+    });
+  }
+
+  return jsi::Value::undefined();
+}
+
+} // namespace vision

android/src/main/cpp/FrameProcessorPluginHostObject.h

@@ -0,0 +1,31 @@
+//
+// Created by Marc Rousavy on 21.07.23.
+//
+
+#pragma once
+
+#include <jsi/jsi.h>
+#include "java-bindings/JFrameProcessorPlugin.h"
+#include <memory>
+#include <ReactCommon/CallInvoker.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+
+class FrameProcessorPluginHostObject: public jsi::HostObject {
+public:
+    explicit FrameProcessorPluginHostObject(jni::global_ref<vision::JFrameProcessorPlugin::javaobject> plugin):
+            _plugin(plugin) { }
+    ~FrameProcessorPluginHostObject() { }
+
+public:
+    std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
+    jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
+
+private:
+    jni::global_ref<vision::JFrameProcessorPlugin::javaobject> _plugin;
+};
+
+} // namespace vision

android/src/main/cpp/JSIJNIConversion.cpp

@@ -21,7 +21,7 @@
 #include <folly/dynamic.h>
 
 #include "FrameHostObject.h"
-#include "java-bindings/JImageProxy.h"
+#include "java-bindings/JFrame.h"
 #include "java-bindings/JArrayList.h"
 #include "java-bindings/JHashMap.h"
 
@@ -178,10 +178,9 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
     auto hashMap = toHashMapFunc(object.get());
     return convertJNIObjectToJSIValue(runtime, hashMap);
 
-  } else if (object->isInstanceOf(JImageProxy::javaClassStatic())) {
-    // ImageProxy
-
-    auto frame = static_ref_cast<JImageProxy>(object);
+  } else if (object->isInstanceOf(JFrame::javaClassStatic())) {
+    // Frame
+    auto frame = static_ref_cast<JFrame>(object);
 
     // box into HostObject
     auto hostObject = std::make_shared<FrameHostObject>(frame);

android/src/main/cpp/VisionCamera.cpp

@@ -1,13 +1,14 @@
 #include <jni.h>
 #include <fbjni/fbjni.h>
 #include "FrameProcessorRuntimeManager.h"
-#include "CameraView.h"
-#include "VisionCameraScheduler.h"
+#include "java-bindings/JCameraView.h"
+#include "java-bindings/JVisionCameraScheduler.h"
+#include "VisionCameraProxy.h"
 
 JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
   return facebook::jni::initialize(vm, [] {
-    vision::FrameProcessorRuntimeManager::registerNatives();
-    vision::CameraView::registerNatives();
-    vision::VisionCameraScheduler::registerNatives();
+    vision::VisionCameraInstaller::registerNatives();
+    vision::JCameraView::registerNatives();
+    vision::JVisionCameraScheduler::registerNatives();
   });
 }

android/src/main/cpp/VisionCameraProxy.cpp

@@ -0,0 +1,172 @@
+//
+// Created by Marc Rousavy on 21.07.23.
+//
+
+#include "VisionCameraProxy.h"
+#include <jsi/jsi.h>
+
+#include <react-native-worklets/WKTJsiWorklet.h>
+#include <react-native-worklets/WKTJsiHostObject.h>
+
+#include "java-bindings/JCameraView.h"
+#include "java-bindings/JFrameProcessor.h"
+#include "java-bindings/JFrameProcessorPlugin.h"
+#include "FrameProcessorPluginHostObject.h"
+#include "FrameHostObject.h"
+#include "JSIJNIConversion.h"
+#include "JSITypedArray.h"
+
+#include <android/log.h>
+
+namespace vision {
+
+using namespace facebook;
+
+VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime,
+                                     std::shared_ptr<react::CallInvoker> callInvoker,
+                                     jni::global_ref<JVisionCameraScheduler::javaobject> scheduler) {
+  _callInvoker = callInvoker;
+  _scheduler = scheduler;
+
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
+
+  auto runOnJS = [callInvoker](std::function<void()>&& f) {
+    // Run on React JS Runtime
+    callInvoker->invokeAsync(std::move(f));
+  };
+  auto runOnWorklet = [this](std::function<void()>&& f) {
+      // Run on Frame Processor Worklet Runtime
+    _scheduler->cthis()->dispatchAsync([f = std::move(f)](){
+        f();
+    });
+  };
+  _workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
+                                                                   &runtime,
+                                                                   runOnJS,
+                                                                   runOnWorklet);
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
+}
+
+VisionCameraProxy::~VisionCameraProxy() {
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
+  // Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
+  vision::invalidateArrayBufferCache(*_workletContext->getJsRuntime());
+  vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime());
+}
+
+std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {
+  std::vector<jsi::PropNameID> result;
+  result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor")));
+  result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor")));
+  result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin")));
+  result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled")));
+  return result;
+}
+
+jni::global_ref<JCameraView::javaobject> VisionCameraProxy::findCameraViewById(int viewId) {
+  // TODO: implement findCameraViewById
+  /*static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod<CameraView(jint)>("findCameraViewById");
+  auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId);
+  return make_global(weakCameraView);*/
+  return nullptr;
+}
+
+void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& object) {
+  auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
+  auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
+
+  auto view = findCameraViewById(viewTag);
+  JFrameProcessor frameProcessor(worklet, _workletContext);
+
+  // TODO: Set frame processor on JCameraView
+}
+
+void VisionCameraProxy::removeFrameProcessor(jsi::Runtime& runtime, int viewTag) {
+  auto view = findCameraViewById(viewTag);
+
+  // TODO: Remove frame processor from JCameraView
+}
+
+jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options) {
+  // TODO: Get Frame Processor Plugin here
+
+  auto pluginHostObject = std::make_shared<FrameProcessorPluginHostObject>(plugin, _callInvoker);
+  return jsi::Object::createFromHostObject(runtime, pluginHostObject);
+}
+
+jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
+  auto name = propName.utf8(runtime);
+
+  if (name == "isSkiaEnabled") {
+#ifdef VISION_CAMERA_ENABLE_SKIA
+    return jsi::Value(true);
+#else
+    return jsi::Value(false);
+#endif
+  }
+  if (name == "setFrameProcessor") {
+    return jsi::Function::createFromHostFunction(runtime,
+                                                 jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
+                                                 1,
+                                                 [this](jsi::Runtime& runtime,
+                                                        const jsi::Value& thisValue,
+                                                        const jsi::Value* arguments,
+                                                        size_t count) -> jsi::Value {
+      auto viewTag = arguments[0].asNumber();
+      auto object = arguments[1].asObject(runtime);
+      this->setFrameProcessor(runtime, static_cast<int>(viewTag), object);
+      return jsi::Value::undefined();
+    });
+  }
+  if (name == "removeFrameProcessor") {
+    return jsi::Function::createFromHostFunction(runtime,
+                                                 jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
+                                                 1,
+                                                 [this](jsi::Runtime& runtime,
+                                                        const jsi::Value& thisValue,
+                                                        const jsi::Value* arguments,
+                                                        size_t count) -> jsi::Value {
+      auto viewTag = arguments[0].asNumber();
+      this->removeFrameProcessor(runtime, static_cast<int>(viewTag));
+      return jsi::Value::undefined();
+    });
+  }
+  if (name == "getFrameProcessorPlugin") {
+    return jsi::Function::createFromHostFunction(runtime,
+                                                 jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
+                                                 1,
+                                                 [this](jsi::Runtime& runtime,
+                                                        const jsi::Value& thisValue,
+                                                        const jsi::Value* arguments,
+                                                        size_t count) -> jsi::Value {
+      if (count < 1 || !arguments[0].isString()) {
+        throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
+      }
+      auto pluginName = arguments[0].asString(runtime).utf8(runtime);
+      auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime);
+
+      return this->getFrameProcessorPlugin(runtime, pluginName, options);
+    });
+  }
+
+  return jsi::Value::undefined();
+}
+
+
+void VisionCameraInstaller::install(jni::alias_ref<jni::JClass>,
+                                    jlong jsiRuntimePtr,
+                                    jni::alias_ref<react::CallInvokerHolder::javaobject> callInvokerHolder,
+                                    jni::alias_ref<JVisionCameraScheduler::javaobject> scheduler) {
+  // cast from JNI hybrid objects to C++ instances
+  jsi::Runtime& runtime = *reinterpret_cast<jsi::Runtime*>(jsiRuntimePtr);
+  std::shared_ptr<react::CallInvoker> callInvoker = callInvokerHolder->cthis()->getCallInvoker();
+  jni::global_ref<JVisionCameraScheduler::javaobject> sharedScheduler = make_global(scheduler);
+
+  // global.VisionCameraProxy
+  auto visionCameraProxy = std::make_shared<VisionCameraProxy>(runtime, callInvoker, sharedScheduler);
+  runtime.global().setProperty(runtime,
+                               "VisionCameraProxy",
+                               jsi::Object::createFromHostObject(runtime, visionCameraProxy));
+}
+
+}