From 0611b736ccdbad381eb0c6ddee23e356aed9cacf Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 17:51:49 +0200 Subject: [PATCH 01/11] First Android rewrite --- android/CMakeLists.txt | 14 +- android/src/main/cpp/FrameHostObject.cpp | 2 +- android/src/main/cpp/FrameHostObject.h | 6 +- .../cpp/FrameProcessorPluginHostObject.cpp | 52 ++++++ .../main/cpp/FrameProcessorPluginHostObject.h | 31 ++++ android/src/main/cpp/JSIJNIConversion.cpp | 9 +- android/src/main/cpp/VisionCamera.cpp | 11 +- android/src/main/cpp/VisionCameraProxy.cpp | 172 ++++++++++++++++++ android/src/main/cpp/VisionCameraProxy.h | 60 ++++++ .../JCameraView.cpp} | 18 +- .../JCameraView.h} | 6 +- android/src/main/cpp/java-bindings/JFrame.cpp | 65 +++++++ .../java-bindings/{JImageProxy.h => JFrame.h} | 6 +- .../cpp/java-bindings/JFrameProcessor.cpp | 28 +++ .../main/cpp/java-bindings/JFrameProcessor.h | 40 ++++ .../java-bindings/JFrameProcessorPlugin.cpp | 8 +- .../cpp/java-bindings/JFrameProcessorPlugin.h | 6 +- .../main/cpp/java-bindings/JImageProxy.cpp | 78 -------- .../JVisionCameraScheduler.cpp} | 18 +- .../JVisionCameraScheduler.h} | 6 +- .../java/com/mrousavy/camera/CameraView.kt | 3 - .../com/mrousavy/camera/CameraViewModule.kt | 56 ++---- .../mrousavy/camera/frameprocessor/Frame.java | 130 +++++++++++++ .../camera/frameprocessor/FrameProcessor.java | 16 ++ .../frameprocessor/FrameProcessorPlugin.java | 2 +- .../frameprocessor/ImageProxyUtils.java | 98 ---------- .../frameprocessor/VisionCameraInstaller.java | 9 + ...RuntimeManager.kt => VisionCameraProxy.kt} | 9 +- .../camera/parsers/PermissionStatus+String.kt | 11 ++ .../FrameProcessorPluginHostObject.mm | 2 + ios/Frame Processor/VisionCameraProxy.mm | 2 +- 31 files changed, 693 insertions(+), 281 deletions(-) create mode 100644 android/src/main/cpp/FrameProcessorPluginHostObject.cpp create mode 100644 android/src/main/cpp/FrameProcessorPluginHostObject.h create mode 100644 android/src/main/cpp/VisionCameraProxy.cpp create mode 100644 android/src/main/cpp/VisionCameraProxy.h rename android/src/main/cpp/{CameraView.cpp => java-bindings/JCameraView.cpp} (66%) rename android/src/main/cpp/{CameraView.h => java-bindings/JCameraView.h} (83%) create mode 100644 android/src/main/cpp/java-bindings/JFrame.cpp rename android/src/main/cpp/java-bindings/{JImageProxy.h => JFrame.h} (73%) create mode 100644 android/src/main/cpp/java-bindings/JFrameProcessor.cpp create mode 100644 android/src/main/cpp/java-bindings/JFrameProcessor.h delete mode 100644 android/src/main/cpp/java-bindings/JImageProxy.cpp rename android/src/main/cpp/{VisionCameraScheduler.cpp => java-bindings/JVisionCameraScheduler.cpp} (52%) rename android/src/main/cpp/{VisionCameraScheduler.h => java-bindings/JVisionCameraScheduler.h} (85%) create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java delete mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java rename android/src/main/java/com/mrousavy/camera/frameprocessor/{FrameProcessorRuntimeManager.kt => VisionCameraProxy.kt} (88%) create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 088f664c98..12e37cf902 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -32,15 +32,17 @@ add_library( ${PACKAGE_NAME} SHARED ../cpp/JSITypedArray.cpp - src/main/cpp/VisionCamera.cpp - src/main/cpp/JSIJNIConversion.cpp src/main/cpp/FrameHostObject.cpp - src/main/cpp/FrameProcessorRuntimeManager.cpp - src/main/cpp/CameraView.cpp - src/main/cpp/VisionCameraScheduler.cpp + src/main/cpp/FrameProcessorPluginHostObject.cpp + src/main/cpp/JSIJNIConversion.cpp + src/main/cpp/VisionCamera.cpp + src/main/cpp/VisionCameraProxy.cpp + src/main/cpp/java-bindings/JCameraView.cpp + src/main/cpp/java-bindings/JFrame.cpp + src/main/cpp/java-bindings/JFrameProcessor.cpp src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp - src/main/cpp/java-bindings/JImageProxy.cpp src/main/cpp/java-bindings/JHashMap.cpp + src/main/cpp/java-bindings/JVisionCameraScheduler.cpp ) # Header Search Paths (includes) diff --git a/android/src/main/cpp/FrameHostObject.cpp b/android/src/main/cpp/FrameHostObject.cpp index 60b367ae12..cb753cf6d8 100644 --- a/android/src/main/cpp/FrameHostObject.cpp +++ b/android/src/main/cpp/FrameHostObject.cpp @@ -18,7 +18,7 @@ namespace vision { using namespace facebook; -FrameHostObject::FrameHostObject(jni::alias_ref image): frame(make_global(image)), _refCount(0) { } +FrameHostObject::FrameHostObject(jni::alias_ref frame): frame(make_global(frame)), _refCount(0) { } FrameHostObject::~FrameHostObject() { // Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread diff --git a/android/src/main/cpp/FrameHostObject.h b/android/src/main/cpp/FrameHostObject.h index eb22db52ac..53d045b117 100644 --- a/android/src/main/cpp/FrameHostObject.h +++ b/android/src/main/cpp/FrameHostObject.h @@ -11,7 +11,7 @@ #include #include -#include "java-bindings/JImageProxy.h" +#include "java-bindings/JFrame.h" namespace vision { @@ -19,7 +19,7 @@ using namespace facebook; class JSI_EXPORT FrameHostObject : public jsi::HostObject { public: - explicit FrameHostObject(jni::alias_ref image); + explicit FrameHostObject(jni::alias_ref frame); ~FrameHostObject(); public: @@ -27,7 +27,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject { std::vector getPropertyNames(jsi::Runtime &rt) override; public: - jni::global_ref frame; + jni::global_ref frame; private: static auto constexpr TAG = "VisionCamera"; diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.cpp b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp new file mode 100644 index 0000000000..f219ca95f0 --- /dev/null +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp @@ -0,0 +1,52 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "FrameProcessorPluginHostObject.h" +#include +#include "FrameHostObject.h" +#include "JSIJNIConversion.h" + +namespace vision { + +using namespace facebook; + +std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime &runtime) { + std::vector result; + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); + return result; +} + +jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime &runtime, const jsi::PropNameID &propName) { + auto name = propName.utf8(runtime); + + if (name == "call") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "call"), + 2, + [=](jsi::Runtime &runtime, + const jsi::Value &thisValue, + const jsi::Value *arguments, + size_t count) -> jsi::Value { + // Frame is first argument + auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime); + auto frame = frameHostObject->frame; + + // Options are second argument (possibly undefined) + jobject options = nullptr; + if (count > 1) { + options = JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[1]); + } + + // Call actual plugin + auto result = _plugin->callback(frame, options); + + // Convert result value to jsi::Value (possibly undefined) + return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result); + }); + } + + return jsi::Value::undefined(); +} + +} // namespace vision \ No newline at end of file diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.h b/android/src/main/cpp/FrameProcessorPluginHostObject.h new file mode 100644 index 0000000000..944becc68e --- /dev/null +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.h @@ -0,0 +1,31 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include +#include "java-bindings/JFrameProcessorPlugin.h" +#include +#include +#include + +namespace vision { + +using namespace facebook; + +class FrameProcessorPluginHostObject: public jsi::HostObject { +public: + explicit FrameProcessorPluginHostObject(jni::global_ref plugin): + _plugin(plugin) { } + ~FrameProcessorPluginHostObject() { } + +public: + std::vector getPropertyNames(jsi::Runtime& runtime) override; + jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; + +private: + jni::global_ref _plugin; +}; + +} // namespace vision \ No newline at end of file diff --git a/android/src/main/cpp/JSIJNIConversion.cpp b/android/src/main/cpp/JSIJNIConversion.cpp index d34308f6a2..879daf19fd 100644 --- a/android/src/main/cpp/JSIJNIConversion.cpp +++ b/android/src/main/cpp/JSIJNIConversion.cpp @@ -21,7 +21,7 @@ #include #include "FrameHostObject.h" -#include "java-bindings/JImageProxy.h" +#include "java-bindings/JFrame.h" #include "java-bindings/JArrayList.h" #include "java-bindings/JHashMap.h" @@ -178,10 +178,9 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c auto hashMap = toHashMapFunc(object.get()); return convertJNIObjectToJSIValue(runtime, hashMap); - } else if (object->isInstanceOf(JImageProxy::javaClassStatic())) { - // ImageProxy - - auto frame = static_ref_cast(object); + } else if (object->isInstanceOf(JFrame::javaClassStatic())) { + // Frame + auto frame = static_ref_cast(object); // box into HostObject auto hostObject = std::make_shared(frame); diff --git a/android/src/main/cpp/VisionCamera.cpp b/android/src/main/cpp/VisionCamera.cpp index 6b91b8ef25..5dc0eca03e 100644 --- a/android/src/main/cpp/VisionCamera.cpp +++ b/android/src/main/cpp/VisionCamera.cpp @@ -1,13 +1,14 @@ #include #include #include "FrameProcessorRuntimeManager.h" -#include "CameraView.h" -#include "VisionCameraScheduler.h" +#include "java-bindings/JCameraView.h" +#include "java-bindings/JVisionCameraScheduler.h" +#include "VisionCameraProxy.h" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { return facebook::jni::initialize(vm, [] { - vision::FrameProcessorRuntimeManager::registerNatives(); - vision::CameraView::registerNatives(); - vision::VisionCameraScheduler::registerNatives(); + vision::VisionCameraInstaller::registerNatives(); + vision::JCameraView::registerNatives(); + vision::JVisionCameraScheduler::registerNatives(); }); } diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp new file mode 100644 index 0000000000..c574de12a9 --- /dev/null +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -0,0 +1,172 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "VisionCameraProxy.h" +#include + +#include +#include + +#include "java-bindings/JCameraView.h" +#include "java-bindings/JFrameProcessor.h" +#include "java-bindings/JFrameProcessorPlugin.h" +#include "FrameProcessorPluginHostObject.h" +#include "FrameHostObject.h" +#include "JSIJNIConversion.h" +#include "JSITypedArray.h" + +#include + +namespace vision { + +using namespace facebook; + +VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime, + std::shared_ptr callInvoker, + jni::global_ref scheduler) { + _callInvoker = callInvoker; + _scheduler = scheduler; + + __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); + + auto runOnJS = [callInvoker](std::function&& f) { + // Run on React JS Runtime + callInvoker->invokeAsync(std::move(f)); + }; + auto runOnWorklet = [this](std::function&& f) { + // Run on Frame Processor Worklet Runtime + _scheduler->cthis()->dispatchAsync([f = std::move(f)](){ + f(); + }); + }; + _workletContext = std::make_shared("VisionCamera", + &runtime, + runOnJS, + runOnWorklet); + __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!"); +} + +VisionCameraProxy::~VisionCameraProxy() { + __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context..."); + // Destroy ArrayBuffer cache for both the JS and the Worklet Runtime. + vision::invalidateArrayBufferCache(*_workletContext->getJsRuntime()); + vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime()); +} + +std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) { + std::vector result; + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled"))); + return result; +} + +jni::global_ref VisionCameraProxy::findCameraViewById(int viewId) { + // TODO: implement findCameraViewById + /*static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod("findCameraViewById"); + auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId); + return make_global(weakCameraView);*/ + return nullptr; +} + +void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& object) { + auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime); + auto worklet = std::make_shared(runtime, object.getProperty(runtime, "frameProcessor")); + + auto view = findCameraViewById(viewTag); + JFrameProcessor frameProcessor(worklet, _workletContext); + + // TODO: Set frame processor on JCameraView +} + +void VisionCameraProxy::removeFrameProcessor(jsi::Runtime& runtime, int viewTag) { + auto view = findCameraViewById(viewTag); + + // TODO: Remove frame processor from JCameraView +} + +jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options) { + // TODO: Get Frame Processor Plugin here + + auto pluginHostObject = std::make_shared(plugin, _callInvoker); + return jsi::Object::createFromHostObject(runtime, pluginHostObject); +} + +jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { + auto name = propName.utf8(runtime); + + if (name == "isSkiaEnabled") { +#ifdef VISION_CAMERA_ENABLE_SKIA + return jsi::Value(true); +#else + return jsi::Value(false); +#endif + } + if (name == "setFrameProcessor") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + auto viewTag = arguments[0].asNumber(); + auto object = arguments[1].asObject(runtime); + this->setFrameProcessor(runtime, static_cast(viewTag), object); + return jsi::Value::undefined(); + }); + } + if (name == "removeFrameProcessor") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + auto viewTag = arguments[0].asNumber(); + this->removeFrameProcessor(runtime, static_cast(viewTag)); + return jsi::Value::undefined(); + }); + } + if (name == "getFrameProcessorPlugin") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + if (count < 1 || !arguments[0].isString()) { + throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); + } + auto pluginName = arguments[0].asString(runtime).utf8(runtime); + auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime); + + return this->getFrameProcessorPlugin(runtime, pluginName, options); + }); + } + + return jsi::Value::undefined(); +} + + +void VisionCameraInstaller::install(jni::alias_ref, + jlong jsiRuntimePtr, + jni::alias_ref callInvokerHolder, + jni::alias_ref scheduler) { + // cast from JNI hybrid objects to C++ instances + jsi::Runtime& runtime = *reinterpret_cast(jsiRuntimePtr); + std::shared_ptr callInvoker = callInvokerHolder->cthis()->getCallInvoker(); + jni::global_ref sharedScheduler = make_global(scheduler); + + // global.VisionCameraProxy + auto visionCameraProxy = std::make_shared(runtime, callInvoker, sharedScheduler); + runtime.global().setProperty(runtime, + "VisionCameraProxy", + jsi::Object::createFromHostObject(runtime, visionCameraProxy)); +} + +} \ No newline at end of file diff --git a/android/src/main/cpp/VisionCameraProxy.h b/android/src/main/cpp/VisionCameraProxy.h new file mode 100644 index 0000000000..b95e5440b8 --- /dev/null +++ b/android/src/main/cpp/VisionCameraProxy.h @@ -0,0 +1,60 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "java-bindings/JVisionCameraScheduler.h" +#include "java-bindings/JCameraView.h" + +namespace vision { + +using namespace facebook; + +class VisionCameraProxy: public jsi::HostObject { +public: + explicit VisionCameraProxy(jsi::Runtime& runtime, + std::shared_ptr callInvoker, + jni::global_ref scheduler); + ~VisionCameraProxy(); + +public: + std::vector getPropertyNames(jsi::Runtime& runtime) override; + jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; + +private: + void setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& frameProcessor); + void removeFrameProcessor(jsi::Runtime& runtime, int viewTag); + jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options); + jni::global_ref findCameraViewById(int viewId); + +private: + std::shared_ptr _workletContext; + std::shared_ptr _callInvoker; + jni::global_ref _scheduler; + static constexpr const char* TAG = "VisionCameraProxy"; +}; + + +class VisionCameraInstaller: public jni::JavaClass { +public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;"; + static void registerNatives() { + javaClassStatic()->registerNatives({ + makeNativeMethod("install", VisionCameraInstaller::install) + }); + } + static void install(jni::alias_ref clazz, + jlong jsiRuntimePtr, + jni::alias_ref callInvoker, + jni::alias_ref scheduler); +}; + +} \ No newline at end of file diff --git a/android/src/main/cpp/CameraView.cpp b/android/src/main/cpp/java-bindings/JCameraView.cpp similarity index 66% rename from android/src/main/cpp/CameraView.cpp rename to android/src/main/cpp/java-bindings/JCameraView.cpp index 665bd312b5..8ef63734eb 100644 --- a/android/src/main/cpp/CameraView.cpp +++ b/android/src/main/cpp/java-bindings/JCameraView.cpp @@ -2,7 +2,7 @@ // Created by Marc Rousavy on 14.06.21. // -#include "CameraView.h" +#include "JCameraView.h" #include #include @@ -17,20 +17,20 @@ namespace vision { using namespace facebook; using namespace jni; -using TSelf = local_ref; +using TSelf = local_ref; -TSelf CameraView::initHybrid(alias_ref jThis) { +TSelf JCameraView::initHybrid(alias_ref jThis) { return makeCxxInstance(jThis); } -void CameraView::registerNatives() { +void JCameraView::registerNatives() { registerHybrid({ - makeNativeMethod("initHybrid", CameraView::initHybrid), - makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback), + makeNativeMethod("initHybrid", JCameraView::initHybrid), + makeNativeMethod("frameProcessorCallback", JCameraView::frameProcessorCallback), }); } -void CameraView::frameProcessorCallback(const alias_ref& frame) { +void JCameraView::frameProcessorCallback(const alias_ref& frame) { if (frameProcessor_ == nullptr) { __android_log_write(ANDROID_LOG_WARN, TAG, "Called Frame Processor callback, but `frameProcessor` is null!"); return; @@ -47,11 +47,11 @@ void CameraView::frameProcessorCallback(const alias_ref } } -void CameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) { +void JCameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) { frameProcessor_ = frameProcessor; } -void vision::CameraView::unsetFrameProcessor() { +void vision::JCameraView::unsetFrameProcessor() { frameProcessor_ = nullptr; } diff --git a/android/src/main/cpp/CameraView.h b/android/src/main/cpp/java-bindings/JCameraView.h similarity index 83% rename from android/src/main/cpp/CameraView.h rename to android/src/main/cpp/java-bindings/JCameraView.h index a1ca070d76..817f20828c 100644 --- a/android/src/main/cpp/CameraView.h +++ b/android/src/main/cpp/java-bindings/JCameraView.h @@ -16,7 +16,7 @@ namespace vision { using namespace facebook; using TFrameProcessor = std::function)>; -class CameraView : public jni::HybridClass { +class JCameraView : public jni::HybridClass { public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;"; static auto constexpr TAG = "VisionCamera"; @@ -29,12 +29,12 @@ class CameraView : public jni::HybridClass { private: friend HybridBase; - jni::global_ref javaPart_; + jni::global_ref javaPart_; TFrameProcessor frameProcessor_; void frameProcessorCallback(const jni::alias_ref& frame); - explicit CameraView(jni::alias_ref jThis) : + explicit JCameraView(jni::alias_ref jThis) : javaPart_(jni::make_global(jThis)), frameProcessor_(nullptr) {} diff --git a/android/src/main/cpp/java-bindings/JFrame.cpp b/android/src/main/cpp/java-bindings/JFrame.cpp new file mode 100644 index 0000000000..24752b0de3 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrame.cpp @@ -0,0 +1,65 @@ +// +// Created by Marc on 21.07.2023. +// + +#include "JFrame.h" + +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +int JFrame::getWidth() const { + static const auto getWidthMethod = getClass()->getMethod("getWidth"); + return getWidthMethod(self()); +} + +int JFrame::getHeight() const { + static const auto getWidthMethod = getClass()->getMethod("getHeight"); + return getWidthMethod(self()); +} + +bool JFrame::getIsValid() const { + static const auto getIsValidMethod = getClass()->getMethod("getIsValid"); + return getIsValidMethod(self()); +} + +bool JFrame::getIsMirrored() const { + static const auto getIsMirroredMethod = getClass()->getMethod("getIsMirrored"); + return getIsMirroredMethod(self()); +} + +jlong JFrame::getTimestamp() const { + static const auto getTimestampMethod = getClass()->getMethod("getTimestamp"); + return getTimestampMethod(self()); +} + +local_ref JFrame::getOrientation() const { + static const auto getOrientationMethod = getClass()->getMethod("getOrientation"); + return getOrientationMethod(self()); +} + +int JFrame::getPlanesCount() const { + static const auto getPlanesCountMethod = getClass()->getMethod("getPlanesCount"); + return getPlanesCountMethod(self()); +} + +int JFrame::getBytesPerRow() const { + static const auto getBytesPerRowMethod = getClass()->getMethod("getBytesPerRow"); + return getBytesPerRowMethod(self()); +} + +local_ref JFrame::toByteArray() const { + static const auto toByteArrayMethodMethod = getClass()->getMethod("toByteArray"); + return toByteArrayMethod(self()); +} + +void JFrame::close() { + static const auto closeMethod = getClass()->getMethod("close"); + closeMethod(self()); +} + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JImageProxy.h b/android/src/main/cpp/java-bindings/JFrame.h similarity index 73% rename from android/src/main/cpp/java-bindings/JImageProxy.h rename to android/src/main/cpp/java-bindings/JFrame.h index 81cb5f25eb..8d9949bcc0 100644 --- a/android/src/main/cpp/java-bindings/JImageProxy.h +++ b/android/src/main/cpp/java-bindings/JFrame.h @@ -1,5 +1,5 @@ // -// Created by Marc on 19/06/2021. +// Created by Marc on 21.07.2023. // #pragma once @@ -12,8 +12,8 @@ namespace vision { using namespace facebook; using namespace jni; -struct JImageProxy : public JavaClass { - static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;"; +struct JFrame : public JavaClass { + static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;"; public: int getWidth() const; diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp new file mode 100644 index 0000000000..23418852f3 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp @@ -0,0 +1,28 @@ +// +// Created by Marc Rousavy on 29.09.21. +// + +#include "JFrameProcessor.h" + +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +local_ref JFrameProcessorPlugin::callback(alias_ref image, + alias_ref> params) const { + auto callbackMethod = getClass()->getMethod("callback"); + + auto result = callbackMethod(self(), image, params); + return make_local(result); +} + +std::string JFrameProcessorPlugin::getName() const { + auto getNameMethod = getClass()->getMethod("getName"); + return getNameMethod(self())->toStdString(); +} + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.h b/android/src/main/cpp/java-bindings/JFrameProcessor.h new file mode 100644 index 0000000000..32b8d1c0f6 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.h @@ -0,0 +1,40 @@ +// +// Created by Marc Rousavy on 29.09.21 +// + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "JFrame.h" + +namespace vision { + +using namespace facebook; + +struct JFrameProcessor : public jni::HybridClass { +public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;"; + static jni::local_ref initHybrid(jni::alias_ref jThis); + static void registerNatives(); + explicit JFrameProcessor(std::shared_ptr worklet, + std::shared_ptr context); + +public: + /** + * Call the JS Frame Processor. + */ + void call(alias_ref frame) const; + +private: + friend HybridBase; + jni::global_ref javaPart_; +}; + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp index 54a162b9ac..92753b43eb 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp @@ -12,13 +12,13 @@ namespace vision { using namespace facebook; using namespace jni; -using TCallback = jobject(alias_ref, alias_ref>); +using TCallback = jobject(alias_ref, alias_ref); -local_ref JFrameProcessorPlugin::callback(alias_ref image, - alias_ref> params) const { +local_ref JFrameProcessorPlugin::callback(alias_ref frame, + alias_ref params) const { auto callbackMethod = getClass()->getMethod("callback"); - auto result = callbackMethod(self(), image, params); + auto result = callbackMethod(self(), frame, params); return make_local(result); } diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index ad66c5d54a..dc16396ef4 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -8,7 +8,7 @@ #include #include -#include "JImageProxy.h" +#include "JFrame.h" namespace vision { @@ -22,8 +22,8 @@ struct JFrameProcessorPlugin : public JavaClass { /** * Call the plugin. */ - local_ref callback(alias_ref image, - alias_ref> params) const; + local_ref callback(alias_ref frame, + alias_ref params) const; /** * Get the user-defined name of the Frame Processor Plugin */ diff --git a/android/src/main/cpp/java-bindings/JImageProxy.cpp b/android/src/main/cpp/java-bindings/JImageProxy.cpp deleted file mode 100644 index 4c7b210e48..0000000000 --- a/android/src/main/cpp/java-bindings/JImageProxy.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by Marc Rousavy on 22.06.21. -// - -#include "JImageProxy.h" - -#include -#include - -namespace vision { - -using namespace facebook; -using namespace jni; - -int JImageProxy::getWidth() const { - static const auto getWidthMethod = getClass()->getMethod("getWidth"); - return getWidthMethod(self()); -} - -int JImageProxy::getHeight() const { - static const auto getWidthMethod = getClass()->getMethod("getHeight"); - return getWidthMethod(self()); -} - -alias_ref getUtilsClass() { - static const auto ImageProxyUtilsClass = findClassStatic("com/mrousavy/camera/frameprocessor/ImageProxyUtils"); - return ImageProxyUtilsClass; -} - -bool JImageProxy::getIsValid() const { - auto utilsClass = getUtilsClass(); - static const auto isImageProxyValidMethod = utilsClass->getStaticMethod("isImageProxyValid"); - return isImageProxyValidMethod(utilsClass, self()); -} - -bool JImageProxy::getIsMirrored() const { - auto utilsClass = getUtilsClass(); - static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod("isImageProxyMirrored"); - return isImageProxyMirroredMethod(utilsClass, self()); -} - -jlong JImageProxy::getTimestamp() const { - auto utilsClass = getUtilsClass(); - static const auto getTimestampMethod = utilsClass->getStaticMethod("getTimestamp"); - return getTimestampMethod(utilsClass, self()); -} - -local_ref JImageProxy::getOrientation() const { - auto utilsClass = getUtilsClass(); - static const auto getOrientationMethod = utilsClass->getStaticMethod("getOrientation"); - return getOrientationMethod(utilsClass, self()); -} - -int JImageProxy::getPlanesCount() const { - auto utilsClass = getUtilsClass(); - static const auto getPlanesCountMethod = utilsClass->getStaticMethod("getPlanesCount"); - return getPlanesCountMethod(utilsClass, self()); -} - -int JImageProxy::getBytesPerRow() const { - auto utilsClass = getUtilsClass(); - static const auto getBytesPerRowMethod = utilsClass->getStaticMethod("getBytesPerRow"); - return getBytesPerRowMethod(utilsClass, self()); -} - -local_ref JImageProxy::toByteArray() const { - auto utilsClass = getUtilsClass(); - - static const auto toByteArrayMethod = utilsClass->getStaticMethod("toByteArray"); - return toByteArrayMethod(utilsClass, self()); -} - -void JImageProxy::close() { - static const auto closeMethod = getClass()->getMethod("close"); - closeMethod(self()); -} - -} // namespace vision diff --git a/android/src/main/cpp/VisionCameraScheduler.cpp b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp similarity index 52% rename from android/src/main/cpp/VisionCameraScheduler.cpp rename to android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp index 790ef07380..6e10502e66 100644 --- a/android/src/main/cpp/VisionCameraScheduler.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp @@ -2,31 +2,31 @@ // Created by Marc Rousavy on 25.07.21. // -#include "VisionCameraScheduler.h" +#include "JVisionCameraScheduler.h" #include namespace vision { using namespace facebook; -using TSelf = jni::local_ref; +using TSelf = jni::local_ref; -TSelf VisionCameraScheduler::initHybrid(jni::alias_ref jThis) { +TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref jThis) { return makeCxxInstance(jThis); } -void VisionCameraScheduler::dispatchAsync(std::function job) { +void JVisionCameraScheduler::dispatchAsync(std::function job) { // 1. add job to queue _jobs.push(job); scheduleTrigger(); } -void VisionCameraScheduler::scheduleTrigger() { +void JVisionCameraScheduler::scheduleTrigger() { // 2. schedule `triggerUI` to be called on the java thread static auto method = javaPart_->getClass()->getMethod("scheduleTrigger"); method(javaPart_.get()); } -void VisionCameraScheduler::trigger() { +void JVisionCameraScheduler::trigger() { std::unique_lock lock(_mutex); // 3. call job we enqueued in step 1. auto job = _jobs.front(); @@ -34,10 +34,10 @@ void VisionCameraScheduler::trigger() { _jobs.pop(); } -void VisionCameraScheduler::registerNatives() { +void JVisionCameraScheduler::registerNatives() { registerHybrid({ - makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid), - makeNativeMethod("trigger", VisionCameraScheduler::trigger), + makeNativeMethod("initHybrid", JVisionCameraScheduler::initHybrid), + makeNativeMethod("trigger", JVisionCameraScheduler::trigger), }); } diff --git a/android/src/main/cpp/VisionCameraScheduler.h b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h similarity index 85% rename from android/src/main/cpp/VisionCameraScheduler.h rename to android/src/main/cpp/java-bindings/JVisionCameraScheduler.h index 59f48e8038..ca4366e12b 100644 --- a/android/src/main/cpp/VisionCameraScheduler.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h @@ -23,7 +23,7 @@ using namespace facebook; * 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there * 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1. */ -class VisionCameraScheduler : public jni::HybridClass { +class JVisionCameraScheduler : public jni::HybridClass { public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;"; static jni::local_ref initHybrid(jni::alias_ref jThis); @@ -34,11 +34,11 @@ class VisionCameraScheduler : public jni::HybridClass { private: friend HybridBase; - jni::global_ref javaPart_; + jni::global_ref javaPart_; std::queue> _jobs; std::mutex _mutex; - explicit VisionCameraScheduler(jni::alias_ref jThis): + explicit JVisionCameraScheduler(jni::alias_ref jThis): javaPart_(jni::make_global(jThis)) {} // Schedules a call to `trigger` on the VisionCamera FP Thread diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index fcd1b82b82..df88eaafac 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -24,15 +24,12 @@ import androidx.lifecycle.* import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.* -import com.facebook.react.uimanager.events.RCTEventEmitter -import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlinx.coroutines.guava.await import java.lang.IllegalArgumentException import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import kotlin.math.floor import kotlin.math.max import kotlin.math.min diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index a229a1c62f..0c889f6e47 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -3,16 +3,11 @@ package com.mrousavy.camera import android.Manifest import android.content.Context import android.content.pm.PackageManager -import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.os.Build import android.util.Log -import android.util.Size -import androidx.camera.core.CameraSelector -import androidx.camera.extensions.ExtensionMode import androidx.camera.extensions.ExtensionsManager import androidx.camera.lifecycle.ProcessCameraProvider -import androidx.camera.video.QualitySelector import androidx.core.content.ContextCompat import com.facebook.react.bridge.* import com.facebook.react.module.annotations.ReactModule @@ -21,7 +16,7 @@ import com.facebook.react.modules.core.PermissionListener import com.facebook.react.uimanager.UIManagerHelper import com.facebook.react.bridge.ReactApplicationContext import java.util.concurrent.ExecutorService -import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager +import com.mrousavy.camera.frameprocessor.VisionCameraProxy import com.mrousavy.camera.parsers.* import com.mrousavy.camera.utils.* import kotlinx.coroutines.* @@ -30,40 +25,23 @@ import java.util.concurrent.Executors @ReactModule(name = CameraViewModule.TAG) @Suppress("unused") -class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) { +class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJavaModule(reactContext) { companion object { const val TAG = "CameraView" var RequestCode = 10 - - fun parsePermissionStatus(status: Int): String { - return when (status) { - PackageManager.PERMISSION_DENIED -> "denied" - PackageManager.PERMISSION_GRANTED -> "authorized" - else -> "not-determined" - } - } } var frameProcessorThread: ExecutorService = Executors.newSingleThreadExecutor() private val coroutineScope = CoroutineScope(Dispatchers.Default) // TODO: or Dispatchers.Main? - private var frameProcessorManager: FrameProcessorRuntimeManager? = null - private fun cleanup() { + override fun invalidate() { + super.invalidate() + frameProcessorThread.shutdown() if (coroutineScope.isActive) { coroutineScope.cancel("CameraViewModule has been destroyed.") } } - override fun onCatalystInstanceDestroy() { - super.onCatalystInstanceDestroy() - cleanup() - } - - override fun invalidate() { - super.invalidate() - cleanup() - } - override fun getName(): String { return TAG } @@ -75,6 +53,18 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase return view ?: throw ViewNotFoundError(viewId) } + @ReactMethod(isBlockingSynchronousMethod = true) + fun installFrameProcessorBindings(): Boolean { + try { + frameProcessorManager = VisionCameraProxy(reactApplicationContext, frameProcessorThread) + frameProcessorManager!!.installBindings() + return true + } catch (e: Error) { + Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e) + return false + } + } + @ReactMethod fun takePhoto(viewTag: Int, options: ReadableMap, promise: Promise) { coroutineScope.launch { @@ -151,18 +141,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase } } - @ReactMethod(isBlockingSynchronousMethod = true) - fun installFrameProcessorBindings(): Boolean { - try { - frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext, frameProcessorThread) - frameProcessorManager!!.installBindings() - return true - } catch (e: Error) { - Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e) - return false - } - } - @ReactMethod fun getAvailableCameraDevices(promise: Promise) { coroutineScope.launch { diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java new file mode 100644 index 0000000000..d5adf5c0b5 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -0,0 +1,130 @@ +package com.mrousavy.camera.frameprocessor; + +import android.annotation.SuppressLint; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.media.Image; + +import androidx.annotation.Keep; +import androidx.camera.core.ImageProxy; + +import com.facebook.jni.HybridData; +import com.facebook.proguard.annotations.DoNotStrip; + +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; + +@SuppressWarnings("JavaJniMissingFunction") // using fbjni here +public class Frame { + @SuppressWarnings({"unused", "FieldCanBeLocal"}) + @DoNotStrip + private final HybridData mHybridData; + private final ImageProxy imageProxy; + + public Frame(ImageProxy imageProxy) { + this.imageProxy = imageProxy; + mHybridData = initHybrid(); + } + + private native HybridData initHybrid(); + + + @SuppressWarnings("unused") + @DoNotStrip + private int getWidth() { + return imageProxy.getWidth(); + } + + @SuppressWarnings("unused") + @DoNotStrip + private int getHeight() { + return imageProxy.getHeight(); + } + + @SuppressWarnings("unused") + @DoNotStrip + private boolean getIsValid() { + try { + @SuppressLint("UnsafeOptInUsageError") + Image image = imageProxy.getImage(); + if (image == null) return false; + // will throw an exception if the image is already closed + image.getCropRect(); + // no exception thrown, image must still be valid. + return true; + } catch (Exception e) { + // exception thrown, image has already been closed. + return false; + } + } + + @SuppressWarnings("unused") + @DoNotStrip + private boolean getIsMirrored() { + Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); + // TODO: Figure out how to get isMirrored from ImageProxy + return false; + } + + @SuppressWarnings("unused") + @DoNotStrip + private long getTimestamp() { + return imageProxy.getImageInfo().getTimestamp(); + } + + @SuppressWarnings("unused") + @DoNotStrip + private String getOrientation() { + int rotation = imageProxy.getImageInfo().getRotationDegrees(); + if (rotation >= 45 && rotation < 135) + return "landscapeRight"; + if (rotation >= 135 && rotation < 225) + return "portraitUpsideDown"; + if (rotation >= 225 && rotation < 315) + return "landscapeLeft"; + return "portrait"; + } + + @SuppressWarnings("unused") + @DoNotStrip + private int getPlanesCount() { + return imageProxy.getPlanes().length; + } + + @SuppressWarnings("unused") + @DoNotStrip + private int getBytesPerRow() { + return imageProxy.getPlanes()[0].getRowStride(); + } + + private static byte[] byteArrayCache; + + @SuppressWarnings("unused") + @DoNotStrip + private byte[] toByteArray() { + switch (imageProxy.getFormat()) { + case ImageFormat.YUV_420_888: + ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); + ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer(); + int ySize = yBuffer.remaining(); + int vuSize = vuBuffer.remaining(); + + if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) { + byteArrayCache = new byte[ySize + vuSize]; + } + + yBuffer.get(byteArrayCache, 0, ySize); + vuBuffer.get(byteArrayCache, ySize, vuSize); + + return byteArrayCache; + default: + throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!"); + } + } + + @SuppressWarnings("unused") + @DoNotStrip + private void close() { + imageProxy.close(); + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java new file mode 100644 index 0000000000..32da301910 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java @@ -0,0 +1,16 @@ +package com.mrousavy.camera.frameprocessor; + +import androidx.annotation.Keep; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.facebook.proguard.annotations.DoNotStrip; + +/** + * Represents a JS Frame Processor + */ +public abstract class FrameProcessor { + /** + * Call the JS Frame Processor function with the given Frame + */ + public native void call(Frame frame); +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java index 98905d5ed7..eaaf636402 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java @@ -48,6 +48,6 @@ protected FrameProcessorPlugin(@NonNull String name) { * @param plugin An instance of a plugin. */ public static void register(@NonNull FrameProcessorPlugin plugin) { - FrameProcessorRuntimeManager.Companion.addPlugin(plugin); + VisionCameraProxy.Companion.addPlugin(plugin); } } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java deleted file mode 100644 index f1cf0a1f3e..0000000000 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.mrousavy.camera.frameprocessor; - -import android.annotation.SuppressLint; -import android.graphics.ImageFormat; -import android.graphics.Matrix; -import android.media.Image; - -import androidx.annotation.Keep; -import androidx.camera.core.ImageProxy; -import com.facebook.proguard.annotations.DoNotStrip; - -import java.nio.ByteBuffer; - -@SuppressWarnings("unused") // used through JNI -@DoNotStrip -@Keep -public class ImageProxyUtils { - @SuppressLint("UnsafeOptInUsageError") - @DoNotStrip - @Keep - public static boolean isImageProxyValid(ImageProxy imageProxy) { - try { - Image image = imageProxy.getImage(); - if (image == null) return false; - // will throw an exception if the image is already closed - image.getCropRect(); - // no exception thrown, image must still be valid. - return true; - } catch (Exception e) { - // exception thrown, image has already been closed. - return false; - } - } - - @DoNotStrip - @Keep - public static boolean isImageProxyMirrored(ImageProxy imageProxy) { - Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); - // TODO: Figure out how to get isMirrored from ImageProxy - return false; - } - - @DoNotStrip - @Keep - public static String getOrientation(ImageProxy imageProxy) { - int rotation = imageProxy.getImageInfo().getRotationDegrees(); - if (rotation >= 45 && rotation < 135) - return "landscapeRight"; - if (rotation >= 135 && rotation < 225) - return "portraitUpsideDown"; - if (rotation >= 225 && rotation < 315) - return "landscapeLeft"; - return "portrait"; - } - - @DoNotStrip - @Keep - public static long getTimestamp(ImageProxy imageProxy) { - return imageProxy.getImageInfo().getTimestamp(); - } - - @DoNotStrip - @Keep - public static int getPlanesCount(ImageProxy imageProxy) { - return imageProxy.getPlanes().length; - } - - @DoNotStrip - @Keep - public static int getBytesPerRow(ImageProxy imageProxy) { - return imageProxy.getPlanes()[0].getRowStride(); - } - - private static byte[] byteArrayCache; - - @DoNotStrip - @Keep - public static byte[] toByteArray(ImageProxy imageProxy) { - switch (imageProxy.getFormat()) { - case ImageFormat.YUV_420_888: - ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); - ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer(); - int ySize = yBuffer.remaining(); - int vuSize = vuBuffer.remaining(); - - if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) { - byteArrayCache = new byte[ySize + vuSize]; - } - - yBuffer.get(byteArrayCache, 0, ySize); - vuBuffer.get(byteArrayCache, ySize, vuSize); - - return byteArrayCache; - default: - throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!"); - } - } -} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java new file mode 100644 index 0000000000..f99e5a8929 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java @@ -0,0 +1,9 @@ +package com.mrousavy.camera.frameprocessor; + +import com.mrousavy.camera.CameraViewModule; + +public class VisionCameraInstaller { + private native void install() { + + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt similarity index 88% rename from android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt rename to android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index b6abf7eb99..e9f57a0c48 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -13,10 +13,9 @@ import java.lang.ref.WeakReference import java.util.concurrent.ExecutorService @Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that. -class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { +class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { companion object { - const val TAG = "FrameProcessorRuntime" - private val Plugins: ArrayList = ArrayList() + const val TAG = "VisionCameraProxy" init { try { @@ -26,10 +25,6 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProces throw e } } - - fun addPlugin(plugin: FrameProcessorPlugin) { - Plugins.add(plugin) - } } @DoNotStrip diff --git a/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt new file mode 100644 index 0000000000..a85a7bc605 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt @@ -0,0 +1,11 @@ +package com.mrousavy.camera.parsers + +import android.content.pm.PackageManager + +fun parsePermissionStatus(status: Int): String { + return when (status) { + PackageManager.PERMISSION_DENIED -> "denied" + PackageManager.PERMISSION_GRANTED -> "authorized" + else -> "not-determined" + } +} diff --git a/ios/Frame Processor/FrameProcessorPluginHostObject.mm b/ios/Frame Processor/FrameProcessorPluginHostObject.mm index a91b5f85c0..ce631ce2c3 100644 --- a/ios/Frame Processor/FrameProcessorPluginHostObject.mm +++ b/ios/Frame Processor/FrameProcessorPluginHostObject.mm @@ -12,6 +12,8 @@ #import "FrameHostObject.h" #import "JSINSObjectConversion.h" +using namespace facebook; + std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) { std::vector result; result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); diff --git a/ios/Frame Processor/VisionCameraProxy.mm b/ios/Frame Processor/VisionCameraProxy.mm index 784570726c..81cecfd87c 100644 --- a/ios/Frame Processor/VisionCameraProxy.mm +++ b/ios/Frame Processor/VisionCameraProxy.mm @@ -177,7 +177,7 @@ - (SkiaRenderer* _Nonnull)getSkiaRenderer; const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { - if (count != 1 || !arguments[0].isString()) { + if (count < 1 || !arguments[0].isString()) { throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); } auto pluginName = arguments[0].asString(runtime).utf8(runtime); From 4a82509902b0c60c36de2676e0604ce76fa0bbe7 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 23:07:42 +0200 Subject: [PATCH 02/11] Rewrite Android C++ backend --- android/CMakeLists.txt | 2 +- .../main/cpp/FrameProcessorPluginHostObject.h | 6 +- .../main/cpp/FrameProcessorRuntimeManager.cpp | 245 ------------------ .../main/cpp/FrameProcessorRuntimeManager.h | 54 ---- android/src/main/cpp/VisionCamera.cpp | 7 +- android/src/main/cpp/VisionCameraProxy.cpp | 104 +++----- android/src/main/cpp/VisionCameraProxy.h | 40 +-- .../main/cpp/java-bindings/JCameraView.cpp | 58 ----- .../src/main/cpp/java-bindings/JCameraView.h | 43 --- android/src/main/cpp/java-bindings/JFrame.cpp | 4 +- .../cpp/java-bindings/JFrameProcessor.cpp | 51 +++- .../main/cpp/java-bindings/JFrameProcessor.h | 19 +- .../java-bindings/JFrameProcessorPlugin.cpp | 5 - .../cpp/java-bindings/JFrameProcessorPlugin.h | 4 - .../cpp/java-bindings/JVisionCameraProxy.cpp | 84 ++++++ .../cpp/java-bindings/JVisionCameraProxy.h | 51 ++++ .../java/com/mrousavy/camera/CameraView.kt | 20 +- .../com/mrousavy/camera/CameraViewModule.kt | 11 +- .../frameprocessor/FrameProcessorPlugin.java | 30 +-- .../FrameProcessorPluginRegistry.java | 38 +++ .../frameprocessor/VisionCameraInstaller.java | 7 +- .../frameprocessor/VisionCameraProxy.kt | 59 +++-- 22 files changed, 341 insertions(+), 601 deletions(-) delete mode 100644 android/src/main/cpp/FrameProcessorRuntimeManager.cpp delete mode 100644 android/src/main/cpp/FrameProcessorRuntimeManager.h delete mode 100644 android/src/main/cpp/java-bindings/JCameraView.cpp delete mode 100644 android/src/main/cpp/java-bindings/JCameraView.h create mode 100644 android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp create mode 100644 android/src/main/cpp/java-bindings/JVisionCameraProxy.h create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 12e37cf902..7b19de132e 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -37,11 +37,11 @@ add_library( src/main/cpp/JSIJNIConversion.cpp src/main/cpp/VisionCamera.cpp src/main/cpp/VisionCameraProxy.cpp - src/main/cpp/java-bindings/JCameraView.cpp src/main/cpp/java-bindings/JFrame.cpp src/main/cpp/java-bindings/JFrameProcessor.cpp src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp src/main/cpp/java-bindings/JHashMap.cpp + src/main/cpp/java-bindings/JVisionCameraProxy.cpp src/main/cpp/java-bindings/JVisionCameraScheduler.cpp ) diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.h b/android/src/main/cpp/FrameProcessorPluginHostObject.h index 944becc68e..1645dd715f 100644 --- a/android/src/main/cpp/FrameProcessorPluginHostObject.h +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.h @@ -16,8 +16,8 @@ using namespace facebook; class FrameProcessorPluginHostObject: public jsi::HostObject { public: - explicit FrameProcessorPluginHostObject(jni::global_ref plugin): - _plugin(plugin) { } + explicit FrameProcessorPluginHostObject(jni::alias_ref plugin): + _plugin(make_global(plugin)) { } ~FrameProcessorPluginHostObject() { } public: @@ -25,7 +25,7 @@ class FrameProcessorPluginHostObject: public jsi::HostObject { jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; private: - jni::global_ref _plugin; + jni::global_ref _plugin; }; } // namespace vision \ No newline at end of file diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp b/android/src/main/cpp/FrameProcessorRuntimeManager.cpp deleted file mode 100644 index 412006e44b..0000000000 --- a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp +++ /dev/null @@ -1,245 +0,0 @@ -// -// Created by Marc Rousavy on 11.06.21. -// - -#include "FrameProcessorRuntimeManager.h" -#include -#include -#include -#include -#include -#include - -#include "CameraView.h" -#include "FrameHostObject.h" -#include "JSIJNIConversion.h" -#include "java-bindings/JImageProxy.h" -#include "java-bindings/JFrameProcessorPlugin.h" -#include "JSITypedArray.h" - -namespace vision { - -// type aliases -using TSelf = local_ref::jhybriddata>; -using TJSCallInvokerHolder = jni::alias_ref; -using TAndroidScheduler = jni::alias_ref; - -FrameProcessorRuntimeManager::FrameProcessorRuntimeManager(jni::alias_ref jThis, - jsi::Runtime* jsRuntime, - std::shared_ptr jsCallInvoker, - std::shared_ptr scheduler) : - javaPart_(jni::make_global(jThis)), - _jsRuntime(jsRuntime) { - auto runOnJS = [jsCallInvoker](std::function&& f) { - // Run on React JS Runtime - jsCallInvoker->invokeAsync(std::move(f)); - }; - auto runOnWorklet = [scheduler](std::function&& f) { - // Run on Frame Processor Worklet Runtime - scheduler->dispatchAsync(std::move(f)); - }; - _workletContext = std::make_shared("VisionCamera", - jsRuntime, - runOnJS, - runOnWorklet); -} - -// JNI binding -void vision::FrameProcessorRuntimeManager::registerNatives() { - registerHybrid({ - makeNativeMethod("initHybrid", - FrameProcessorRuntimeManager::initHybrid), - makeNativeMethod("installJSIBindings", - FrameProcessorRuntimeManager::installJSIBindings), - makeNativeMethod("registerPlugin", - FrameProcessorRuntimeManager::registerPlugin), - }); -} - -// JNI init -TSelf vision::FrameProcessorRuntimeManager::initHybrid( - alias_ref jThis, - jlong jsRuntimePointer, - TJSCallInvokerHolder jsCallInvokerHolder, - TAndroidScheduler androidScheduler) { - __android_log_write(ANDROID_LOG_INFO, TAG, - "Initializing FrameProcessorRuntimeManager..."); - - // cast from JNI hybrid objects to C++ instances - auto jsRuntime = reinterpret_cast(jsRuntimePointer); - auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); - auto scheduler = std::shared_ptr(androidScheduler->cthis()); - - return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, scheduler); -} - -global_ref FrameProcessorRuntimeManager::findCameraViewById(int viewId) { - static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod("findCameraViewById"); - auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId); - return make_global(weakCameraView); -} - -void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) { - if (!_workletContext) { - return; - } - // Call console.error() on JS Thread - _workletContext->invokeOnJsThread([message](jsi::Runtime& runtime) { - auto consoleError = runtime - .global() - .getPropertyAsObject(runtime, "console") - .getPropertyAsFunction(runtime, "error"); - consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message)); - }); -} - -void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime, - int viewTag, - const jsi::Value& frameProcessor) { - __android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor..."); - - if (!_workletContext) { - throw jsi::JSError(runtime, - "setFrameProcessor(..): VisionCamera's Worklet Context is not yet initialized!"); - } - - // find camera view - auto cameraView = findCameraViewById(viewTag); - - // convert jsi::Function to a Worklet (can be shared across runtimes) - auto worklet = std::make_shared(runtime, frameProcessor); - auto workletInvoker = std::make_shared(worklet); - - _workletContext->invokeOnWorkletThread([=](RNWorklet::JsiWorkletContext*, jsi::Runtime& rt) { - // Set Frame Processor as callable C++ lambda - this will then call the Worklet - cameraView->cthis()->setFrameProcessor([this, workletInvoker, &rt](jni::alias_ref frame) { - try { - // create HostObject which holds the Frame (JImageProxy) - auto frameHostObject = std::make_shared(frame); - auto argument = jsi::Object::createFromHostObject(rt, frameHostObject); - jsi::Value jsValue(std::move(argument)); - // Call the Worklet on the Worklet Runtime - workletInvoker->call(rt, jsi::Value::undefined(), &jsValue, 1); - } catch (jsi::JSError& jsError) { - // Worklet threw a JS Error, catch it and log it to JS. - auto message = "Frame Processor threw an error: " + jsError.getMessage(); - __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str()); - this->logErrorToJS(message); - } - }); - }); -} - -void FrameProcessorRuntimeManager::unsetFrameProcessor(int viewTag) { - __android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor..."); - - // find camera view - auto cameraView = findCameraViewById(viewTag); - - // call Java method to unset frame processor - cameraView->cthis()->unsetFrameProcessor(); -} - -// actual JSI installer -void FrameProcessorRuntimeManager::installJSIBindings() { - __android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings..."); - - if (_jsRuntime == nullptr) { - __android_log_write(ANDROID_LOG_ERROR, TAG, - "JS-Runtime was null, Frame Processor JSI bindings could not be installed!"); - return; - } - - auto& jsiRuntime = *_jsRuntime; - - // HostObject that attaches the cache to the lifecycle of the Runtime. On Runtime destroy, we destroy the cache. - auto propNameCacheObject = std::make_shared(jsiRuntime); - jsiRuntime.global().setProperty(jsiRuntime, - "__visionCameraPropNameCache", - jsi::Object::createFromHostObject(jsiRuntime, propNameCacheObject)); - - auto setFrameProcessor = JSI_HOST_FUNCTION_LAMBDA { - __android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor..."); - - double viewTag = arguments[0].asNumber(); - const jsi::Value& frameProcessor = arguments[1]; - this->setFrameProcessor(runtime, static_cast(viewTag), frameProcessor); - - return jsi::Value::undefined(); - }; - jsiRuntime.global().setProperty(jsiRuntime, - "setFrameProcessor", - jsi::Function::createFromHostFunction( - jsiRuntime, - jsi::PropNameID::forAscii(jsiRuntime, - "setFrameProcessor"), - 2, // viewTag, frameProcessor - setFrameProcessor)); - - - auto unsetFrameProcessor = JSI_HOST_FUNCTION_LAMBDA { - __android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor..."); - - auto viewTag = arguments[0].asNumber(); - this->unsetFrameProcessor(static_cast(viewTag)); - - return jsi::Value::undefined(); - }; - jsiRuntime.global().setProperty(jsiRuntime, - "unsetFrameProcessor", - jsi::Function::createFromHostFunction( - jsiRuntime, - jsi::PropNameID::forAscii(jsiRuntime, - "unsetFrameProcessor"), - 1, // viewTag - unsetFrameProcessor)); - - __android_log_write(ANDROID_LOG_INFO, TAG, "Finished installing JSI bindings!"); -} - -void FrameProcessorRuntimeManager::registerPlugin(alias_ref plugin) { - auto& runtime = *_jsRuntime; - - // we need a strong reference on the plugin, make_global does that. - auto pluginGlobal = make_global(plugin); - auto pluginName = pluginGlobal->getName(); - - __android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", pluginName.c_str()); - - if (!runtime.global().hasProperty(runtime, "FrameProcessorPlugins")) { - runtime.global().setProperty(runtime, "FrameProcessorPlugins", jsi::Object(runtime)); - } - jsi::Object frameProcessorPlugins = runtime.global().getPropertyAsObject(runtime, "FrameProcessorPlugins"); - - auto function = [pluginGlobal](jsi::Runtime& runtime, - const jsi::Value& thisValue, - const jsi::Value* arguments, - size_t count) -> jsi::Value { - // Unbox object and get typed HostObject - auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime); - auto frameHostObject = dynamic_cast(boxedHostObject.get()); - - // parse params - we are offset by `1` because the frame is the first parameter. - auto params = JArrayClass::newArray(count - 1); - for (size_t i = 1; i < count; i++) { - params->setElement(i - 1, JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[i])); - } - - // call implemented virtual method - auto result = pluginGlobal->callback(frameHostObject->frame, params); - - // convert result from JNI to JSI value - return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result); - }; - - // Assign it to the Proxy. - // A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)" - frameProcessorPlugins.setProperty(runtime, - pluginName.c_str(), - jsi::Function::createFromHostFunction(runtime, - jsi::PropNameID::forAscii(runtime, pluginName), - 1, // frame - function)); -} - -} // namespace vision diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.h b/android/src/main/cpp/FrameProcessorRuntimeManager.h deleted file mode 100644 index 450670ed98..0000000000 --- a/android/src/main/cpp/FrameProcessorRuntimeManager.h +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by Marc Rousavy on 11.06.21. -// - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "CameraView.h" -#include "VisionCameraScheduler.h" -#include "java-bindings/JFrameProcessorPlugin.h" - -namespace vision { - -using namespace facebook; - -class FrameProcessorRuntimeManager : public jni::HybridClass { - public: - static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager;"; - static auto constexpr TAG = "VisionCamera"; - static jni::local_ref initHybrid(jni::alias_ref jThis, - jlong jsContext, - jni::alias_ref jsCallInvokerHolder, - jni::alias_ref androidScheduler); - static void registerNatives(); - - explicit FrameProcessorRuntimeManager(jni::alias_ref jThis, - jsi::Runtime* jsRuntime, - std::shared_ptr jsCallInvoker, - std::shared_ptr scheduler); - - private: - friend HybridBase; - jni::global_ref javaPart_; - jsi::Runtime* _jsRuntime; - std::shared_ptr _workletContext; - - jni::global_ref findCameraViewById(int viewId); - void installJSIBindings(); - void registerPlugin(alias_ref plugin); - void logErrorToJS(const std::string& message); - - void setFrameProcessor(jsi::Runtime& runtime, - int viewTag, - const jsi::Value& frameProcessor); - void unsetFrameProcessor(int viewTag); -}; - -} // namespace vision diff --git a/android/src/main/cpp/VisionCamera.cpp b/android/src/main/cpp/VisionCamera.cpp index 5dc0eca03e..1da9d50b4e 100644 --- a/android/src/main/cpp/VisionCamera.cpp +++ b/android/src/main/cpp/VisionCamera.cpp @@ -1,14 +1,15 @@ #include #include -#include "FrameProcessorRuntimeManager.h" -#include "java-bindings/JCameraView.h" #include "java-bindings/JVisionCameraScheduler.h" +#include "java-bindings/JFrameProcessor.h" +#include "java-bindings/JVisionCameraProxy.h" #include "VisionCameraProxy.h" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { return facebook::jni::initialize(vm, [] { vision::VisionCameraInstaller::registerNatives(); - vision::JCameraView::registerNatives(); + vision::JFrameProcessor::registerNatives(); + vision::JVisionCameraProxy::registerNatives(); vision::JVisionCameraScheduler::registerNatives(); }); } diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp index c574de12a9..2765b61196 100644 --- a/android/src/main/cpp/VisionCameraProxy.cpp +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -5,53 +5,30 @@ #include "VisionCameraProxy.h" #include -#include -#include - -#include "java-bindings/JCameraView.h" #include "java-bindings/JFrameProcessor.h" #include "java-bindings/JFrameProcessorPlugin.h" -#include "FrameProcessorPluginHostObject.h" -#include "FrameHostObject.h" #include "JSIJNIConversion.h" -#include "JSITypedArray.h" #include +#include + +#include "JSITypedArray.h" +#include "FrameProcessorPluginHostObject.h" namespace vision { using namespace facebook; -VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime, - std::shared_ptr callInvoker, - jni::global_ref scheduler) { - _callInvoker = callInvoker; - _scheduler = scheduler; - - __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); - - auto runOnJS = [callInvoker](std::function&& f) { - // Run on React JS Runtime - callInvoker->invokeAsync(std::move(f)); - }; - auto runOnWorklet = [this](std::function&& f) { - // Run on Frame Processor Worklet Runtime - _scheduler->cthis()->dispatchAsync([f = std::move(f)](){ - f(); - }); - }; - _workletContext = std::make_shared("VisionCamera", - &runtime, - runOnJS, - runOnWorklet); - __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!"); +VisionCameraProxy::VisionCameraProxy(jni::alias_ref javaProxy) { + _javaProxy = make_global(javaProxy); } VisionCameraProxy::~VisionCameraProxy() { __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context..."); // Destroy ArrayBuffer cache for both the JS and the Worklet Runtime. - vision::invalidateArrayBufferCache(*_workletContext->getJsRuntime()); - vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime()); + auto workletContext = _javaProxy->cthis()->getWorkletContext(); + invalidateArrayBufferCache(*workletContext->getJsRuntime()); + invalidateArrayBufferCache(workletContext->getWorkletRuntime()); } std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) { @@ -63,34 +40,37 @@ std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& r return result; } -jni::global_ref VisionCameraProxy::findCameraViewById(int viewId) { - // TODO: implement findCameraViewById - /*static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod("findCameraViewById"); - auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId); - return make_global(weakCameraView);*/ - return nullptr; -} - -void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& object) { +void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) { auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime); auto worklet = std::make_shared(runtime, object.getProperty(runtime, "frameProcessor")); + auto workletContext = _javaProxy->cthis()->getWorkletContext(); + + jni::local_ref frameProcessor; + if (frameProcessorType == "frame-processor") { + frameProcessor = JFrameProcessor::create(worklet, workletContext); + } else if (frameProcessorType == "skia-frame-processor") { +#if VISION_CAMERA_ENABLE_SKIA + throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!"); +#else + throw std::runtime_error("system/skia-unavailable: Skia is not installed!"); +#endif + } else { + throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType); + } - auto view = findCameraViewById(viewTag); - JFrameProcessor frameProcessor(worklet, _workletContext); - - // TODO: Set frame processor on JCameraView + _javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor)); } -void VisionCameraProxy::removeFrameProcessor(jsi::Runtime& runtime, int viewTag) { - auto view = findCameraViewById(viewTag); - - // TODO: Remove frame processor from JCameraView +void VisionCameraProxy::removeFrameProcessor(int viewTag) { + _javaProxy->cthis()->removeFrameProcessor(viewTag); } -jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options) { - // TODO: Get Frame Processor Plugin here +jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, jsi::Value jsOptions) { + auto options = JSIJNIConversion::convertJSIValueToJNIObject(runtime, jsOptions); - auto pluginHostObject = std::make_shared(plugin, _callInvoker); + auto plugin = _javaProxy->cthis()->getFrameProcessorPlugin(name, options); + + auto pluginHostObject = std::make_shared(plugin); return jsi::Object::createFromHostObject(runtime, pluginHostObject); } @@ -114,7 +94,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& size_t count) -> jsi::Value { auto viewTag = arguments[0].asNumber(); auto object = arguments[1].asObject(runtime); - this->setFrameProcessor(runtime, static_cast(viewTag), object); + this->setFrameProcessor(static_cast(viewTag), runtime, object); return jsi::Value::undefined(); }); } @@ -127,7 +107,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& const jsi::Value* arguments, size_t count) -> jsi::Value { auto viewTag = arguments[0].asNumber(); - this->removeFrameProcessor(runtime, static_cast(viewTag)); + this->removeFrameProcessor(static_cast(viewTag)); return jsi::Value::undefined(); }); } @@ -143,9 +123,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); } auto pluginName = arguments[0].asString(runtime).utf8(runtime); - auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime); + auto options = jsi::Value(runtime, arguments[1]); - return this->getFrameProcessorPlugin(runtime, pluginName, options); + return this->getFrameProcessorPlugin(runtime, pluginName, std::move(options)); }); } @@ -154,16 +134,10 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& void VisionCameraInstaller::install(jni::alias_ref, - jlong jsiRuntimePtr, - jni::alias_ref callInvokerHolder, - jni::alias_ref scheduler) { - // cast from JNI hybrid objects to C++ instances - jsi::Runtime& runtime = *reinterpret_cast(jsiRuntimePtr); - std::shared_ptr callInvoker = callInvokerHolder->cthis()->getCallInvoker(); - jni::global_ref sharedScheduler = make_global(scheduler); - + jni::alias_ref proxy) { // global.VisionCameraProxy - auto visionCameraProxy = std::make_shared(runtime, callInvoker, sharedScheduler); + auto visionCameraProxy = std::make_shared(proxy); + jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime(); runtime.global().setProperty(runtime, "VisionCameraProxy", jsi::Object::createFromHostObject(runtime, visionCameraProxy)); diff --git a/android/src/main/cpp/VisionCameraProxy.h b/android/src/main/cpp/VisionCameraProxy.h index b95e5440b8..434a59d2de 100644 --- a/android/src/main/cpp/VisionCameraProxy.h +++ b/android/src/main/cpp/VisionCameraProxy.h @@ -4,15 +4,10 @@ #pragma once -#include -#include #include -#include -#include -#include #include "java-bindings/JVisionCameraScheduler.h" -#include "java-bindings/JCameraView.h" +#include "java-bindings/JVisionCameraProxy.h" namespace vision { @@ -20,9 +15,7 @@ using namespace facebook; class VisionCameraProxy: public jsi::HostObject { public: - explicit VisionCameraProxy(jsi::Runtime& runtime, - std::shared_ptr callInvoker, - jni::global_ref scheduler); + explicit VisionCameraProxy(jni::alias_ref javaProxy); ~VisionCameraProxy(); public: @@ -30,31 +23,26 @@ class VisionCameraProxy: public jsi::HostObject { jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; private: - void setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& frameProcessor); - void removeFrameProcessor(jsi::Runtime& runtime, int viewTag); - jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options); - jni::global_ref findCameraViewById(int viewId); + void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor); + void removeFrameProcessor(int viewTag); + jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, jsi::Value options); private: - std::shared_ptr _workletContext; - std::shared_ptr _callInvoker; - jni::global_ref _scheduler; + jni::global_ref _javaProxy; static constexpr const char* TAG = "VisionCameraProxy"; }; class VisionCameraInstaller: public jni::JavaClass { public: - static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;"; - static void registerNatives() { - javaClassStatic()->registerNatives({ - makeNativeMethod("install", VisionCameraInstaller::install) - }); - } - static void install(jni::alias_ref clazz, - jlong jsiRuntimePtr, - jni::alias_ref callInvoker, - jni::alias_ref scheduler); + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;"; + static void registerNatives() { + javaClassStatic()->registerNatives({ + makeNativeMethod("install", VisionCameraInstaller::install) + }); + } + static void install(jni::alias_ref clazz, + jni::alias_ref proxy); }; } \ No newline at end of file diff --git a/android/src/main/cpp/java-bindings/JCameraView.cpp b/android/src/main/cpp/java-bindings/JCameraView.cpp deleted file mode 100644 index 8ef63734eb..0000000000 --- a/android/src/main/cpp/java-bindings/JCameraView.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by Marc Rousavy on 14.06.21. -// - -#include "JCameraView.h" - -#include -#include -#include - -#include -#include -#include - -namespace vision { - -using namespace facebook; -using namespace jni; - -using TSelf = local_ref; - -TSelf JCameraView::initHybrid(alias_ref jThis) { - return makeCxxInstance(jThis); -} - -void JCameraView::registerNatives() { - registerHybrid({ - makeNativeMethod("initHybrid", JCameraView::initHybrid), - makeNativeMethod("frameProcessorCallback", JCameraView::frameProcessorCallback), - }); -} - -void JCameraView::frameProcessorCallback(const alias_ref& frame) { - if (frameProcessor_ == nullptr) { - __android_log_write(ANDROID_LOG_WARN, TAG, "Called Frame Processor callback, but `frameProcessor` is null!"); - return; - } - - try { - frameProcessor_(frame); - } catch (const jsi::JSError& error) { - // TODO: jsi::JSErrors cannot be caught on Hermes. They crash the entire app. - auto stack = std::regex_replace(error.getStack(), std::regex("\n"), "\n "); - __android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw an error! %s\nIn: %s", error.getMessage().c_str(), stack.c_str()); - } catch (const std::exception& exception) { - __android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw a C++ error! %s", exception.what()); - } -} - -void JCameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) { - frameProcessor_ = frameProcessor; -} - -void vision::JCameraView::unsetFrameProcessor() { - frameProcessor_ = nullptr; -} - -} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JCameraView.h b/android/src/main/cpp/java-bindings/JCameraView.h deleted file mode 100644 index 817f20828c..0000000000 --- a/android/src/main/cpp/java-bindings/JCameraView.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Created by Marc Rousavy on 14.06.21. -// - -#pragma once - -#include -#include - -#include - -#include "java-bindings/JImageProxy.h" - -namespace vision { - -using namespace facebook; -using TFrameProcessor = std::function)>; - -class JCameraView : public jni::HybridClass { - public: - static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;"; - static auto constexpr TAG = "VisionCamera"; - static jni::local_ref initHybrid(jni::alias_ref jThis); - static void registerNatives(); - - // TODO: Use template<> to avoid heap allocation for std::function<> - void setFrameProcessor(const TFrameProcessor&& frameProcessor); - void unsetFrameProcessor(); - - private: - friend HybridBase; - jni::global_ref javaPart_; - TFrameProcessor frameProcessor_; - - void frameProcessorCallback(const jni::alias_ref& frame); - - explicit JCameraView(jni::alias_ref jThis) : - javaPart_(jni::make_global(jThis)), - frameProcessor_(nullptr) - {} -}; - -} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrame.cpp b/android/src/main/cpp/java-bindings/JFrame.cpp index 24752b0de3..e87cd349f4 100644 --- a/android/src/main/cpp/java-bindings/JFrame.cpp +++ b/android/src/main/cpp/java-bindings/JFrame.cpp @@ -23,12 +23,12 @@ int JFrame::getHeight() const { } bool JFrame::getIsValid() const { - static const auto getIsValidMethod = getClass()->getMethod("getIsValid"); + static const auto getIsValidMethod = getClass()->getMethod("getIsValid"); return getIsValidMethod(self()); } bool JFrame::getIsMirrored() const { - static const auto getIsMirroredMethod = getClass()->getMethod("getIsMirrored"); + static const auto getIsMirroredMethod = getClass()->getMethod("getIsMirrored"); return getIsMirroredMethod(self()); } diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp index 23418852f3..408001f194 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp @@ -6,23 +6,58 @@ #include #include +#include "JFrame.h" namespace vision { using namespace facebook; using namespace jni; -local_ref JFrameProcessorPlugin::callback(alias_ref image, - alias_ref> params) const { - auto callbackMethod = getClass()->getMethod("callback"); +void JFrameProcessor::registerNatives() { + registerHybrid({ + makeNativeMethod("call", JFrameProcessor::call) + }); +} + +using TSelf = jni::local_ref; + +JFrameProcessor::JFrameProcessor(std::shared_ptr worklet, + std::shared_ptr context) { + _workletContext = context; + _workletInvoker = std::make_shared(worklet); +} + +TSelf JFrameProcessor::create(std::shared_ptr worklet, + std::shared_ptr context) { + return JFrameProcessor::newObjectCxxArgs(worklet, context); +} - auto result = callbackMethod(self(), image, params); - return make_local(result); +void JFrameProcessor::callWithFrameHostObject(std::shared_ptr frameHostObject) const { + // Call the Frame Processor on the Worklet Runtime + jsi::Runtime& runtime = _workletContext->getWorkletRuntime(); + + try { + // Wrap HostObject as JSI Value + auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject); + jsi::Value jsValue(std::move(argument)); + + // Call the Worklet with the Frame JS Host Object as an argument + _workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1); + } catch (jsi::JSError& jsError) { + // JS Error occured, print it to console. + const std::string& message = jsError.getMessage(); + + _workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) { + auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error"); + logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message)); + }); + } } -std::string JFrameProcessorPlugin::getName() const { - auto getNameMethod = getClass()->getMethod("getName"); - return getNameMethod(self())->toStdString(); +void JFrameProcessor::call(jni::alias_ref frame) { + // Create the Frame Host Object wrapping the internal Frame + auto frameHostObject = std::make_shared(frame); + callWithFrameHostObject(frameHostObject); } } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.h b/android/src/main/cpp/java-bindings/JFrameProcessor.h index 32b8d1c0f6..e9c2463e7a 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessor.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.h @@ -13,6 +13,7 @@ #include #include "JFrame.h" +#include "FrameHostObject.h" namespace vision { @@ -21,20 +22,28 @@ using namespace facebook; struct JFrameProcessor : public jni::HybridClass { public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;"; - static jni::local_ref initHybrid(jni::alias_ref jThis); static void registerNatives(); - explicit JFrameProcessor(std::shared_ptr worklet, - std::shared_ptr context); + static jni::local_ref create(std::shared_ptr worklet, + std::shared_ptr context); public: /** * Call the JS Frame Processor. */ - void call(alias_ref frame) const; + void call(alias_ref frame); + +private: + // Private constructor. Use `create(..)` to create new instances. + explicit JFrameProcessor(std::shared_ptr worklet, + std::shared_ptr context); + +private: + void callWithFrameHostObject(std::shared_ptr frameHostObject) const; private: friend HybridBase; - jni::global_ref javaPart_; + std::shared_ptr _workletInvoker; + std::shared_ptr _workletContext; }; } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp index 92753b43eb..591a7fcb7e 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp @@ -22,9 +22,4 @@ local_ref JFrameProcessorPlugin::callback(alias_ref return make_local(result); } -std::string JFrameProcessorPlugin::getName() const { - auto getNameMethod = getClass()->getMethod("getName"); - return getNameMethod(self())->toStdString(); -} - } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index dc16396ef4..81e8741c6b 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -24,10 +24,6 @@ struct JFrameProcessorPlugin : public JavaClass { */ local_ref callback(alias_ref frame, alias_ref params) const; - /** - * Get the user-defined name of the Frame Processor Plugin - */ - std::string getName() const; }; } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp new file mode 100644 index 0000000000..070add49a4 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp @@ -0,0 +1,84 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "JVisionCameraProxy.h" + +#include +#include +#include +#include +#include "FrameProcessorPluginHostObject.h" + +namespace vision { + +using TSelf = local_ref::jhybriddata>; +using TJSCallInvokerHolder = jni::alias_ref; +using TScheduler = jni::alias_ref; + +JVisionCameraProxy::JVisionCameraProxy(jni::alias_ref javaThis, + jsi::Runtime* runtime, + std::shared_ptr callInvoker, + jni::global_ref scheduler) { + _javaPart = make_global(javaThis); + + __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); + + auto runOnJS = [callInvoker](std::function&& f) { + // Run on React JS Runtime + callInvoker->invokeAsync(std::move(f)); + }; + auto runOnWorklet = [scheduler](std::function&& f) { + // Run on Frame Processor Worklet Runtime + scheduler->cthis()->dispatchAsync([f = std::move(f)](){ + f(); + }); + }; + _workletContext = std::make_shared("VisionCamera", + runtime, + runOnJS, + runOnWorklet); + __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!"); +} + + + +void JVisionCameraProxy::setFrameProcessor(int viewTag, + alias_ref frameProcessor) { + auto setFrameProcessorMethod = javaClassLocal()->getMethod)>("setFrameProcessor"); + setFrameProcessorMethod(_javaPart, viewTag, frameProcessor); +} + +void JVisionCameraProxy::removeFrameProcessor(int viewTag) { + auto removeFrameProcessorMethod = javaClassLocal()->getMethod("removeFrameProcessor"); + removeFrameProcessorMethod(_javaPart, viewTag); +} + +local_ref JVisionCameraProxy::getFrameProcessorPlugin(std::string name, + jobject options) { + auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, jobject)>("setFrameProcessor"); + return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), options); +} + +void JVisionCameraProxy::registerNatives() { + registerHybrid({ + makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid) + }); +} + +TSelf JVisionCameraProxy::initHybrid( + alias_ref jThis, + jlong jsRuntimePointer, + TJSCallInvokerHolder jsCallInvokerHolder, + TScheduler scheduler) { + __android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy..."); + + // cast from JNI hybrid objects to C++ instances + auto jsRuntime = reinterpret_cast(jsRuntimePointer); + auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); + auto sharedScheduler = make_global(scheduler); + + return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler); +} + +} \ No newline at end of file diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h new file mode 100644 index 0000000000..a5cc19dc53 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h @@ -0,0 +1,51 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include +#include +#include + +#include "JFrameProcessorPlugin.h" +#include "JVisionCameraScheduler.h" +#include "JFrameProcessor.h" + +namespace vision { + +using namespace facebook; + +class JVisionCameraProxy : public jni::HybridClass { +public: + static void registerNatives(); + + void setFrameProcessor(int viewTag, + jni::alias_ref frameProcessor); + void removeFrameProcessor(int viewTag); + jni::local_ref getFrameProcessorPlugin(std::string name, + jobject options); + +public: + std::shared_ptr getWorkletContext() { return _workletContext; } + +private: + std::shared_ptr _workletContext; + +private: + friend HybridBase; + jni::global_ref _javaPart; + static auto constexpr TAG = "VisionCameraProxy"; + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;"; + + explicit JVisionCameraProxy(jni::alias_ref javaThis, + jsi::Runtime* jsRuntime, + std::shared_ptr jsCallInvoker, + jni::global_ref scheduler); + static jni::local_ref initHybrid(jni::alias_ref javaThis, + jlong jsRuntimePointer, + jni::alias_ref jsCallInvokerHolder, + jni::alias_ref scheduler); +}; + +} diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index df88eaafac..60116b17e4 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -24,6 +24,8 @@ import androidx.lifecycle.* import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.* +import com.mrousavy.camera.frameprocessor.Frame +import com.mrousavy.camera.frameprocessor.FrameProcessor import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlinx.coroutines.guava.await @@ -115,8 +117,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer internal var camera: Camera? = null internal var imageCapture: ImageCapture? = null internal var videoCapture: VideoCapture? = null - private var imageAnalysis: ImageAnalysis? = null + public var frameProcessor: FrameProcessor? = null private var preview: Preview? = null + private var imageAnalysis: ImageAnalysis? = null internal var activeVideoRecording: Recording? = null @@ -153,10 +156,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer private var minZoom: Float = 1f private var maxZoom: Float = 1f - @DoNotStrip - private var mHybridData: HybridData? = null - - @Suppress("LiftReturnOrAssignment", "RedundantIf") + @Suppress("RedundantIf") internal val fallbackToSnapshot: Boolean @SuppressLint("UnsafeOptInUsageError") get() { @@ -187,8 +187,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer } init { - mHybridData = initHybrid() - previewView = PreviewView(context) previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) previewView.installHierarchyFitter() // If this is not called correctly, view finder will be black/blank @@ -241,9 +239,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer imageAnalysis?.targetRotation = outputRotation } - private external fun initHybrid(): HybridData - private external fun frameProcessorCallback(frame: ImageProxy) - override fun getLifecycle(): Lifecycle { return lifecycleRegistry } @@ -458,8 +453,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer imageAnalysis = imageAnalysisBuilder.build().apply { setAnalyzer(cameraExecutor) { image -> // Call JS Frame Processor - frameProcessorCallback(image) - // frame gets closed in FrameHostObject implementation (JS ref counting) + val frame = Frame(image) + frameProcessor?.call(frame) + // ...frame gets closed in FrameHostObject implementation via JS ref counting } } useCases.add(imageAnalysis!!) diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index 0c889f6e47..5a7a8750c3 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -15,6 +15,7 @@ import com.facebook.react.modules.core.PermissionAwareActivity import com.facebook.react.modules.core.PermissionListener import com.facebook.react.uimanager.UIManagerHelper import com.facebook.react.bridge.ReactApplicationContext +import com.mrousavy.camera.frameprocessor.VisionCameraInstaller import java.util.concurrent.ExecutorService import com.mrousavy.camera.frameprocessor.VisionCameraProxy import com.mrousavy.camera.parsers.* @@ -55,13 +56,13 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod(isBlockingSynchronousMethod = true) fun installFrameProcessorBindings(): Boolean { - try { - frameProcessorManager = VisionCameraProxy(reactApplicationContext, frameProcessorThread) - frameProcessorManager!!.installBindings() - return true + return try { + val proxy = VisionCameraProxy(reactApplicationContext, frameProcessorThread) + VisionCameraInstaller.install(proxy) + true } catch (e: Error) { Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e) - return false + false } } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java index eaaf636402..c9e1a44736 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java @@ -12,8 +12,6 @@ @DoNotStrip @Keep public abstract class FrameProcessorPlugin { - private final @NonNull String mName; - /** * The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives. * @param image The CameraX ImageProxy. Don't call .close() on this, as VisionCamera handles that. @@ -23,31 +21,5 @@ public abstract class FrameProcessorPlugin { */ @DoNotStrip @Keep - public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object[] params); - - /** - * Initializes the native plugin part. - * @param name Specifies the Frame Processor Plugin's name in the Runtime. - * The actual name in the JS Runtime will be prefixed with two underscores (`__`) - */ - protected FrameProcessorPlugin(@NonNull String name) { - mName = name; - } - - /** - * Get the user-defined name of the Frame Processor Plugin. - */ - @DoNotStrip - @Keep - public @NonNull String getName() { - return mName; - } - - /** - * Registers the given plugin in the Frame Processor Runtime. - * @param plugin An instance of a plugin. - */ - public static void register(@NonNull FrameProcessorPlugin plugin) { - VisionCameraProxy.Companion.addPlugin(plugin); - } + public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object params); } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java new file mode 100644 index 0000000000..675c01114a --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java @@ -0,0 +1,38 @@ +package com.mrousavy.camera.frameprocessor; + +import androidx.annotation.Keep; + +import com.facebook.proguard.annotations.DoNotStrip; + +import java.util.HashMap; +import java.util.Map; + +@DoNotStrip +@Keep +public class FrameProcessorPluginRegistry { + private static final Map _plugins = new HashMap<>(); + + @DoNotStrip + @Keep + public static void addFrameProcessorPlugin(String name, PluginInitializer pluginInitializer) throws Exception { + if (_plugins.containsKey(name)) { + throw new Exception("Tried to add a Frame Processor Plugin with a name that already exists! " + + "Either choose unique names, or remove the unused plugin. Name: " + name); + } + _plugins.put(name, pluginInitializer); + } + + @DoNotStrip + @Keep + public static FrameProcessorPlugin getPlugin(String name, Object options) { + PluginInitializer initializer = _plugins.get(name); + if (initializer == null) { + return null; + } + return initializer.initializePlugin(options); + } + + public interface PluginInitializer { + FrameProcessorPlugin initializePlugin(Object options); + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java index f99e5a8929..23833c367c 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java @@ -1,9 +1,6 @@ package com.mrousavy.camera.frameprocessor; -import com.mrousavy.camera.CameraViewModule; - +@SuppressWarnings("JavaJniMissingFunction") // we use fbjni public class VisionCameraInstaller { - private native void install() { - - } + public static native void install(VisionCameraProxy proxy); } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index e9f57a0c48..f65610fe20 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -12,11 +12,11 @@ import com.mrousavy.camera.ViewNotFoundError import java.lang.ref.WeakReference import java.util.concurrent.ExecutorService -@Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that. + +@Suppress("KotlinJniMissingFunction") // we use fbjni. class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { companion object { const val TAG = "VisionCameraProxy" - init { try { System.loadLibrary("VisionCamera") @@ -26,47 +26,50 @@ class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: } } } - - @DoNotStrip - private var mHybridData: HybridData? = null - private var mContext: WeakReference? = null - private var mScheduler: VisionCameraScheduler? = null + private var _hybridData: HybridData + private var _context: WeakReference + private var _scheduler: VisionCameraScheduler init { val jsCallInvokerHolder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl val jsRuntimeHolder = context.javaScriptContextHolder.get() - mScheduler = VisionCameraScheduler(frameProcessorThread) - mContext = WeakReference(context) - mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler!!) + _scheduler = VisionCameraScheduler(frameProcessorThread) + _context = WeakReference(context) + _hybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, _scheduler) } - @Suppress("unused") - @DoNotStrip - @Keep - fun findCameraViewById(viewId: Int): CameraView { + private fun findCameraViewById(viewId: Int): CameraView { Log.d(TAG, "Finding view $viewId...") - val ctx = mContext?.get() + val ctx = _context.get() val view = if (ctx != null) UIManagerHelper.getUIManager(ctx, viewId)?.resolveView(viewId) as CameraView? else null Log.d(TAG, if (view != null) "Found view $viewId!" else "Couldn't find view $viewId!") return view ?: throw ViewNotFoundError(viewId) } - fun installBindings() { - Log.i(TAG, "Installing JSI Bindings on JS Thread...") - installJSIBindings() - Log.i(TAG, "Installing Frame Processor Plugins...") - Plugins.forEach { plugin -> - registerPlugin(plugin) - } - Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!") + @DoNotStrip + @Keep + fun setFrameProcessor(viewId: Int, frameProcessor: FrameProcessor) { + val view = findCameraViewById(viewId) + view.frameProcessor = frameProcessor + } + + @DoNotStrip + @Keep + fun removeFrameProcessor(viewId: Int) { + val view = findCameraViewById(viewId) + view.frameProcessor = null + } + + @DoNotStrip + @Keep + fun getFrameProcessorPlugin(name: String, options: Any): FrameProcessorPlugin { + return FrameProcessorPluginRegistry.getPlugin(name, options) } // private C++ funcs private external fun initHybrid( - jsContext: Long, - jsCallInvokerHolder: CallInvokerHolderImpl, - scheduler: VisionCameraScheduler + jsContext: Long, + jsCallInvokerHolder: CallInvokerHolderImpl, + scheduler: VisionCameraScheduler ): HybridData - private external fun registerPlugin(plugin: FrameProcessorPlugin) - private external fun installJSIBindings() } From def0921756f07c50af266a5b771bed434c08c564 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 23:34:15 +0200 Subject: [PATCH 03/11] Pass `ReadableNativeMap`, fix build error --- .../cpp/FrameProcessorPluginHostObject.cpp | 8 +-- android/src/main/cpp/JSIJNIConversion.cpp | 72 +------------------ android/src/main/cpp/JSIJNIConversion.h | 3 +- android/src/main/cpp/VisionCameraProxy.cpp | 8 +-- android/src/main/cpp/VisionCameraProxy.h | 2 +- android/src/main/cpp/java-bindings/JFrame.cpp | 2 +- .../java-bindings/JFrameProcessorPlugin.cpp | 2 +- .../cpp/java-bindings/JFrameProcessorPlugin.h | 3 +- .../cpp/java-bindings/JVisionCameraProxy.cpp | 6 +- .../cpp/java-bindings/JVisionCameraProxy.h | 3 +- .../camera/frameprocessor/FrameProcessor.java | 1 + .../frameprocessor/FrameProcessorPlugin.java | 3 +- .../FrameProcessorPluginRegistry.java | 20 +++--- .../frameprocessor/VisionCameraProxy.kt | 19 ++--- .../example/ExampleFrameProcessorPlugin.java | 18 +++-- .../camera/example/MainApplication.java | 7 +- 16 files changed, 67 insertions(+), 110 deletions(-) diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.cpp b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp index f219ca95f0..fce813d062 100644 --- a/android/src/main/cpp/FrameProcessorPluginHostObject.cpp +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp @@ -11,13 +11,13 @@ namespace vision { using namespace facebook; -std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime &runtime) { +std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) { std::vector result; result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); return result; } -jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime &runtime, const jsi::PropNameID &propName) { +jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { auto name = propName.utf8(runtime); if (name == "call") { @@ -33,9 +33,9 @@ jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime &runtime, const jsi: auto frame = frameHostObject->frame; // Options are second argument (possibly undefined) - jobject options = nullptr; + local_ref options = nullptr; if (count > 1) { - options = JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[1]); + options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime)); } // Call actual plugin diff --git a/android/src/main/cpp/JSIJNIConversion.cpp b/android/src/main/cpp/JSIJNIConversion.cpp index 879daf19fd..5655a1bb71 100644 --- a/android/src/main/cpp/JSIJNIConversion.cpp +++ b/android/src/main/cpp/JSIJNIConversion.cpp @@ -29,75 +29,9 @@ namespace vision { using namespace facebook; -jobject JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtime &runtime, const jsi::Value &value) { - if (value.isBool()) { - // jsi::Bool - - auto boolean = jni::JBoolean::valueOf(value.getBool()); - return boolean.release(); - - } else if (value.isNumber()) { - // jsi::Number - - auto number = jni::JDouble::valueOf(value.getNumber()); - return number.release(); - - } else if (value.isNull() || value.isUndefined()) { - // jsi::undefined - - return nullptr; - - } else if (value.isString()) { - // jsi::String - - auto string = jni::make_jstring(value.getString(runtime).utf8(runtime)); - return string.release(); - - } else if (value.isObject()) { - // jsi::Object - - auto object = value.asObject(runtime); - - if (object.isArray(runtime)) { - // jsi::Array - - auto dynamic = jsi::dynamicFromValue(runtime, value); - auto nativeArray = react::ReadableNativeArray::newObjectCxxArgs(std::move(dynamic)); - return nativeArray.release(); - - } else if (object.isHostObject(runtime)) { - // jsi::HostObject - - auto boxedHostObject = object.getHostObject(runtime); - auto hostObject = dynamic_cast(boxedHostObject.get()); - if (hostObject != nullptr) { - // return jni local_ref to the JImageProxy - return hostObject->frame.get(); - } else { - // it's different kind of HostObject. We don't support it. - throw std::runtime_error("Received an unknown HostObject! Cannot convert to a JNI value."); - } - - } else if (object.isFunction(runtime)) { - // jsi::Function - - // TODO: Convert Function to Callback - throw std::runtime_error("Cannot convert a JS Function to a JNI value (yet)!"); - - } else { - // jsi::Object - - auto dynamic = jsi::dynamicFromValue(runtime, value); - auto map = react::ReadableNativeMap::createWithContents(std::move(dynamic)); - return map.release(); - } - } else { - // unknown jsi type! - - auto stringRepresentation = value.toString(runtime).utf8(runtime); - auto message = "Received unknown JSI value! (" + stringRepresentation + ") Cannot convert to a JNI value."; - throw std::runtime_error(message); - } +jni::local_ref JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) { + auto dynamic = jsi::dynamicFromValue(runtime, jsi::Value(runtime, object)); + return react::ReadableNativeMap::createWithContents(std::move(dynamic)); } jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref& object) { diff --git a/android/src/main/cpp/JSIJNIConversion.h b/android/src/main/cpp/JSIJNIConversion.h index 5f031ec0c6..3e55848c7c 100644 --- a/android/src/main/cpp/JSIJNIConversion.h +++ b/android/src/main/cpp/JSIJNIConversion.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace vision { @@ -14,7 +15,7 @@ namespace JSIJNIConversion { using namespace facebook; -jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); +jni::local_ref convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object); jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref& object); diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp index 2765b61196..07f032ae79 100644 --- a/android/src/main/cpp/VisionCameraProxy.cpp +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -65,8 +65,8 @@ void VisionCameraProxy::removeFrameProcessor(int viewTag) { _javaProxy->cthis()->removeFrameProcessor(viewTag); } -jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, jsi::Value jsOptions) { - auto options = JSIJNIConversion::convertJSIValueToJNIObject(runtime, jsOptions); +jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& jsOptions) { + auto options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, jsOptions); auto plugin = _javaProxy->cthis()->getFrameProcessorPlugin(name, options); @@ -123,9 +123,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); } auto pluginName = arguments[0].asString(runtime).utf8(runtime); - auto options = jsi::Value(runtime, arguments[1]); + auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime); - return this->getFrameProcessorPlugin(runtime, pluginName, std::move(options)); + return this->getFrameProcessorPlugin(runtime, pluginName, options); }); } diff --git a/android/src/main/cpp/VisionCameraProxy.h b/android/src/main/cpp/VisionCameraProxy.h index 434a59d2de..edff2cf893 100644 --- a/android/src/main/cpp/VisionCameraProxy.h +++ b/android/src/main/cpp/VisionCameraProxy.h @@ -25,7 +25,7 @@ class VisionCameraProxy: public jsi::HostObject { private: void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor); void removeFrameProcessor(int viewTag); - jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, jsi::Value options); + jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options); private: jni::global_ref _javaProxy; diff --git a/android/src/main/cpp/java-bindings/JFrame.cpp b/android/src/main/cpp/java-bindings/JFrame.cpp index e87cd349f4..d79658dc97 100644 --- a/android/src/main/cpp/java-bindings/JFrame.cpp +++ b/android/src/main/cpp/java-bindings/JFrame.cpp @@ -53,7 +53,7 @@ int JFrame::getBytesPerRow() const { } local_ref JFrame::toByteArray() const { - static const auto toByteArrayMethodMethod = getClass()->getMethod("toByteArray"); + static const auto toByteArrayMethod = getClass()->getMethod("toByteArray"); return toByteArrayMethod(self()); } diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp index 591a7fcb7e..0c8e2b54b2 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp @@ -15,7 +15,7 @@ using namespace jni; using TCallback = jobject(alias_ref, alias_ref); local_ref JFrameProcessorPlugin::callback(alias_ref frame, - alias_ref params) const { + alias_ref params) const { auto callbackMethod = getClass()->getMethod("callback"); auto result = callbackMethod(self(), frame, params); diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index 81e8741c6b..068bad3a7a 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "JFrame.h" @@ -23,7 +24,7 @@ struct JFrameProcessorPlugin : public JavaClass { * Call the plugin. */ local_ref callback(alias_ref frame, - alias_ref params) const; + alias_ref params) const; }; } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp index 070add49a4..5cecfd775b 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp @@ -9,12 +9,14 @@ #include #include #include "FrameProcessorPluginHostObject.h" +#include namespace vision { using TSelf = local_ref::jhybriddata>; using TJSCallInvokerHolder = jni::alias_ref; using TScheduler = jni::alias_ref; +using TOptions = jni::local_ref; JVisionCameraProxy::JVisionCameraProxy(jni::alias_ref javaThis, jsi::Runtime* runtime, @@ -55,8 +57,8 @@ void JVisionCameraProxy::removeFrameProcessor(int viewTag) { } local_ref JVisionCameraProxy::getFrameProcessorPlugin(std::string name, - jobject options) { - auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, jobject)>("setFrameProcessor"); + TOptions options) { + auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, TOptions)>("setFrameProcessor"); return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), options); } diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h index a5cc19dc53..0285b7141b 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "JFrameProcessorPlugin.h" #include "JVisionCameraScheduler.h" @@ -24,7 +25,7 @@ class JVisionCameraProxy : public jni::HybridClass { jni::alias_ref frameProcessor); void removeFrameProcessor(int viewTag); jni::local_ref getFrameProcessorPlugin(std::string name, - jobject options); + jni::local_ref options); public: std::shared_ptr getWorkletContext() { return _workletContext; } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java index 32da301910..b67d17dead 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java @@ -8,6 +8,7 @@ /** * Represents a JS Frame Processor */ +@SuppressWarnings("JavaJniMissingFunction") // we're using fbjni. public abstract class FrameProcessor { /** * Call the JS Frame Processor function with the given Frame diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java index c9e1a44736..5ed2adcca2 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java @@ -5,6 +5,7 @@ import androidx.annotation.Nullable; import androidx.camera.core.ImageProxy; import com.facebook.proguard.annotations.DoNotStrip; +import com.facebook.react.bridge.ReadableNativeMap; /** * Declares a Frame Processor Plugin. @@ -21,5 +22,5 @@ public abstract class FrameProcessorPlugin { */ @DoNotStrip @Keep - public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object params); + public abstract @Nullable Object callback(@NonNull ImageProxy image, @Nullable ReadableNativeMap params); } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java index 675c01114a..ecbd2a2ea8 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java @@ -1,8 +1,10 @@ package com.mrousavy.camera.frameprocessor; import androidx.annotation.Keep; +import androidx.annotation.Nullable; import com.facebook.proguard.annotations.DoNotStrip; +import com.facebook.react.bridge.ReadableNativeMap; import java.util.HashMap; import java.util.Map; @@ -10,22 +12,20 @@ @DoNotStrip @Keep public class FrameProcessorPluginRegistry { - private static final Map _plugins = new HashMap<>(); + private static final Map Plugins = new HashMap<>(); @DoNotStrip @Keep - public static void addFrameProcessorPlugin(String name, PluginInitializer pluginInitializer) throws Exception { - if (_plugins.containsKey(name)) { - throw new Exception("Tried to add a Frame Processor Plugin with a name that already exists! " + - "Either choose unique names, or remove the unused plugin. Name: " + name); - } - _plugins.put(name, pluginInitializer); + public static void addFrameProcessorPlugin(String name, PluginInitializer pluginInitializer) { + assert !Plugins.containsKey(name) : "Tried to add a Frame Processor Plugin with a name that already exists! " + + "Either choose unique names, or remove the unused plugin. Name: "; + Plugins.put(name, pluginInitializer); } @DoNotStrip @Keep - public static FrameProcessorPlugin getPlugin(String name, Object options) { - PluginInitializer initializer = _plugins.get(name); + public static FrameProcessorPlugin getPlugin(String name, ReadableNativeMap options) { + PluginInitializer initializer = Plugins.get(name); if (initializer == null) { return null; } @@ -33,6 +33,6 @@ public static FrameProcessorPlugin getPlugin(String name, Object options) { } public interface PluginInitializer { - FrameProcessorPlugin initializePlugin(Object options); + FrameProcessorPlugin initializePlugin(@Nullable ReadableNativeMap options); } } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index f65610fe20..246a745794 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -5,6 +5,7 @@ import androidx.annotation.Keep import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReadableNativeMap import com.facebook.react.turbomodule.core.CallInvokerHolderImpl import com.facebook.react.uimanager.UIManagerHelper import com.mrousavy.camera.CameraView @@ -26,21 +27,23 @@ class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: } } } - private var _hybridData: HybridData - private var _context: WeakReference - private var _scheduler: VisionCameraScheduler + @DoNotStrip + @Keep + private var mHybridData: HybridData + private var mContext: WeakReference + private var mScheduler: VisionCameraScheduler init { val jsCallInvokerHolder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl val jsRuntimeHolder = context.javaScriptContextHolder.get() - _scheduler = VisionCameraScheduler(frameProcessorThread) - _context = WeakReference(context) - _hybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, _scheduler) + mScheduler = VisionCameraScheduler(frameProcessorThread) + mContext = WeakReference(context) + mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler) } private fun findCameraViewById(viewId: Int): CameraView { Log.d(TAG, "Finding view $viewId...") - val ctx = _context.get() + val ctx = mContext.get() val view = if (ctx != null) UIManagerHelper.getUIManager(ctx, viewId)?.resolveView(viewId) as CameraView? else null Log.d(TAG, if (view != null) "Found view $viewId!" else "Couldn't find view $viewId!") return view ?: throw ViewNotFoundError(viewId) @@ -62,7 +65,7 @@ class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: @DoNotStrip @Keep - fun getFrameProcessorPlugin(name: String, options: Any): FrameProcessorPlugin { + fun getFrameProcessorPlugin(name: String, options: ReadableNativeMap): FrameProcessorPlugin { return FrameProcessorPluginRegistry.getPlugin(name, options) } diff --git a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java index f5b8c30c58..a33e7607e9 100644 --- a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java +++ b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java @@ -2,18 +2,26 @@ import android.util.Log; import androidx.camera.core.ImageProxy; + +import com.facebook.react.bridge.ReadableNativeMap; import com.facebook.react.bridge.WritableNativeArray; import com.facebook.react.bridge.WritableNativeMap; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.HashMap; public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { @Override - public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) { - Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.length + " parameters:"); + public Object callback(@NotNull ImageProxy image, @Nullable ReadableNativeMap params) { + HashMap hashMap = params != null ? params.toHashMap() : new HashMap<>(); - for (Object param : params) { - Log.d("ExamplePlugin", " -> " + (param == null ? "(null)" : param.toString() + " (" + param.getClass().getName() + ")")); + Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + hashMap.size() + " parameters:"); + + for (String key : hashMap.keySet()) { + Object value = hashMap.get(key); + Log.d("ExamplePlugin", " -> " + (value == null ? "(null)" : value.toString() + " (" + value.getClass().getName() + ")")); } WritableNativeMap map = new WritableNativeMap(); @@ -31,6 +39,6 @@ public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) { } ExampleFrameProcessorPlugin() { - super("example_plugin"); + } } diff --git a/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java b/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java index 14f79ce54e..ba7e6fb3c2 100644 --- a/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java +++ b/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java @@ -1,10 +1,14 @@ package com.mrousavy.camera.example; import android.app.Application; + +import androidx.annotation.Nullable; + import com.facebook.react.PackageList; import com.facebook.react.ReactApplication; import com.facebook.react.ReactNativeHost; import com.facebook.react.ReactPackage; +import com.facebook.react.bridge.ReadableNativeMap; import com.facebook.soloader.SoLoader; import java.util.List; import com.facebook.react.defaults.DefaultNewArchitectureEntryPoint; @@ -12,6 +16,7 @@ import com.mrousavy.camera.CameraPackage; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; +import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry; public class MainApplication extends Application implements ReactApplication { @@ -61,6 +66,6 @@ public void onCreate() { DefaultNewArchitectureEntryPoint.load(); } - FrameProcessorPlugin.register(new ExampleFrameProcessorPlugin()); + FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin()); } } From 8efb1cc5ceff8e40575c8b5a707adad34a6ceff0 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 23:38:33 +0200 Subject: [PATCH 04/11] fix: Fix FrameProcessor init --- .../mrousavy/camera/frameprocessor/Frame.java | 14 -------------- .../camera/frameprocessor/FrameProcessor.java | 12 +++++++++++- example/src/CameraPage.tsx | 18 ++++++------------ 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index d5adf5c0b5..f8f85da3a0 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -4,31 +4,17 @@ import android.graphics.ImageFormat; import android.graphics.Matrix; import android.media.Image; - -import androidx.annotation.Keep; import androidx.camera.core.ImageProxy; - -import com.facebook.jni.HybridData; import com.facebook.proguard.annotations.DoNotStrip; - import java.nio.ByteBuffer; -import java.util.concurrent.ExecutorService; -@SuppressWarnings("JavaJniMissingFunction") // using fbjni here public class Frame { - @SuppressWarnings({"unused", "FieldCanBeLocal"}) - @DoNotStrip - private final HybridData mHybridData; private final ImageProxy imageProxy; public Frame(ImageProxy imageProxy) { this.imageProxy = imageProxy; - mHybridData = initHybrid(); } - private native HybridData initHybrid(); - - @SuppressWarnings("unused") @DoNotStrip private int getWidth() { diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java index b67d17dead..c89cf7bcee 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java @@ -3,15 +3,25 @@ import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; + +import com.facebook.jni.HybridData; import com.facebook.proguard.annotations.DoNotStrip; /** * Represents a JS Frame Processor */ @SuppressWarnings("JavaJniMissingFunction") // we're using fbjni. -public abstract class FrameProcessor { +public final class FrameProcessor { /** * Call the JS Frame Processor function with the given Frame */ public native void call(Frame frame); + + @DoNotStrip + @Keep + private HybridData mHybridData; + + public FrameProcessor(HybridData hybridData) { + mHybridData = hybridData; + } } diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index fbc6be3e0e..e4e22882eb 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -8,6 +8,7 @@ import { PhotoFile, sortFormats, useCameraDevices, + useFrameProcessor, useSkiaFrameProcessor, VideoFile, } from 'react-native-vision-camera'; @@ -218,15 +219,10 @@ export function CameraPage({ navigation }: Props): React.ReactElement { paint.setImageFilter(imageFilter); const isIOS = Platform.OS === 'ios'; - const frameProcessor = useSkiaFrameProcessor( - (frame) => { - 'worklet'; - console.log(`Width: ${frame.width}`); - - if (frame.isDrawable) frame.render(paint); - }, - [isIOS, paint], - ); + const frameProcessor = useFrameProcessor((frame) => { + 'worklet'; + console.log(`Width: ${frame.width}`); + }, []); return ( @@ -247,12 +243,10 @@ export function CameraPage({ navigation }: Props): React.ReactElement { onError={onError} enableZoomGesture={false} animatedProps={cameraAnimatedProps} - photo={true} - video={true} audio={hasMicrophonePermission} enableFpsGraph={true} orientation="portrait" - frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined} + frameProcessor={frameProcessor} /> From ce0cc916faddbc04cb4bd1115608c94053482644 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 23:46:52 +0200 Subject: [PATCH 05/11] Make a bunch of stuff const reference to avoid copies --- android/src/main/cpp/FrameHostObject.cpp | 2 +- android/src/main/cpp/FrameHostObject.h | 2 +- android/src/main/cpp/VisionCameraProxy.cpp | 6 +++-- android/src/main/cpp/VisionCameraProxy.h | 4 ++-- .../cpp/java-bindings/JFrameProcessor.cpp | 10 +++++---- .../main/cpp/java-bindings/JFrameProcessor.h | 6 ++--- .../java-bindings/JFrameProcessorPlugin.cpp | 6 ++--- .../cpp/java-bindings/JFrameProcessorPlugin.h | 4 ++-- .../cpp/java-bindings/JVisionCameraProxy.cpp | 17 +++++++------- .../cpp/java-bindings/JVisionCameraProxy.h | 12 +++++----- .../java-bindings/JVisionCameraScheduler.cpp | 2 +- .../java-bindings/JVisionCameraScheduler.h | 2 +- .../mrousavy/camera/frameprocessor/Frame.java | 22 +++++++++++-------- .../frameprocessor/FrameProcessorPlugin.java | 5 ++--- .../example/ExampleFrameProcessorPlugin.java | 5 ++++- example/src/CameraPage.tsx | 5 ++++- 16 files changed, 62 insertions(+), 48 deletions(-) diff --git a/android/src/main/cpp/FrameHostObject.cpp b/android/src/main/cpp/FrameHostObject.cpp index cb753cf6d8..b20ccac17f 100644 --- a/android/src/main/cpp/FrameHostObject.cpp +++ b/android/src/main/cpp/FrameHostObject.cpp @@ -18,7 +18,7 @@ namespace vision { using namespace facebook; -FrameHostObject::FrameHostObject(jni::alias_ref frame): frame(make_global(frame)), _refCount(0) { } +FrameHostObject::FrameHostObject(const jni::alias_ref& frame): frame(make_global(frame)), _refCount(0) { } FrameHostObject::~FrameHostObject() { // Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread diff --git a/android/src/main/cpp/FrameHostObject.h b/android/src/main/cpp/FrameHostObject.h index 53d045b117..fc4f5214a2 100644 --- a/android/src/main/cpp/FrameHostObject.h +++ b/android/src/main/cpp/FrameHostObject.h @@ -19,7 +19,7 @@ using namespace facebook; class JSI_EXPORT FrameHostObject : public jsi::HostObject { public: - explicit FrameHostObject(jni::alias_ref frame); + explicit FrameHostObject(const jni::alias_ref& frame); ~FrameHostObject(); public: diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp index 07f032ae79..1ffdf31e44 100644 --- a/android/src/main/cpp/VisionCameraProxy.cpp +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -19,7 +19,7 @@ namespace vision { using namespace facebook; -VisionCameraProxy::VisionCameraProxy(jni::alias_ref javaProxy) { +VisionCameraProxy::VisionCameraProxy(const jni::alias_ref& javaProxy) { _javaProxy = make_global(javaProxy); } @@ -65,7 +65,9 @@ void VisionCameraProxy::removeFrameProcessor(int viewTag) { _javaProxy->cthis()->removeFrameProcessor(viewTag); } -jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& jsOptions) { +jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, + const std::string& name, + const jsi::Object& jsOptions) { auto options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, jsOptions); auto plugin = _javaProxy->cthis()->getFrameProcessorPlugin(name, options); diff --git a/android/src/main/cpp/VisionCameraProxy.h b/android/src/main/cpp/VisionCameraProxy.h index edff2cf893..fe001c6105 100644 --- a/android/src/main/cpp/VisionCameraProxy.h +++ b/android/src/main/cpp/VisionCameraProxy.h @@ -15,7 +15,7 @@ using namespace facebook; class VisionCameraProxy: public jsi::HostObject { public: - explicit VisionCameraProxy(jni::alias_ref javaProxy); + explicit VisionCameraProxy(const jni::alias_ref& javaProxy); ~VisionCameraProxy(); public: @@ -25,7 +25,7 @@ class VisionCameraProxy: public jsi::HostObject { private: void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor); void removeFrameProcessor(int viewTag); - jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options); + jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options); private: jni::global_ref _javaProxy; diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp index 408001f194..9c6aa2c0d4 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp @@ -6,6 +6,8 @@ #include #include + +#include #include "JFrame.h" namespace vision { @@ -23,16 +25,16 @@ using TSelf = jni::local_ref; JFrameProcessor::JFrameProcessor(std::shared_ptr worklet, std::shared_ptr context) { - _workletContext = context; + _workletContext = std::move(context); _workletInvoker = std::make_shared(worklet); } -TSelf JFrameProcessor::create(std::shared_ptr worklet, - std::shared_ptr context) { +TSelf JFrameProcessor::create(const std::shared_ptr& worklet, + const std::shared_ptr& context) { return JFrameProcessor::newObjectCxxArgs(worklet, context); } -void JFrameProcessor::callWithFrameHostObject(std::shared_ptr frameHostObject) const { +void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr& frameHostObject) const { // Call the Frame Processor on the Worklet Runtime jsi::Runtime& runtime = _workletContext->getWorkletRuntime(); diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.h b/android/src/main/cpp/java-bindings/JFrameProcessor.h index e9c2463e7a..b6c1bad108 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessor.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.h @@ -23,8 +23,8 @@ struct JFrameProcessor : public jni::HybridClass { public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;"; static void registerNatives(); - static jni::local_ref create(std::shared_ptr worklet, - std::shared_ptr context); + static jni::local_ref create(const std::shared_ptr& worklet, + const std::shared_ptr& context); public: /** @@ -38,7 +38,7 @@ struct JFrameProcessor : public jni::HybridClass { std::shared_ptr context); private: - void callWithFrameHostObject(std::shared_ptr frameHostObject) const; + void callWithFrameHostObject(const std::shared_ptr& frameHostObject) const; private: friend HybridBase; diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp index 0c8e2b54b2..9fa1d45205 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp @@ -12,10 +12,10 @@ namespace vision { using namespace facebook; using namespace jni; -using TCallback = jobject(alias_ref, alias_ref); +using TCallback = jobject(alias_ref, alias_ref params); -local_ref JFrameProcessorPlugin::callback(alias_ref frame, - alias_ref params) const { +local_ref JFrameProcessorPlugin::callback(const alias_ref& frame, + const alias_ref& params) const { auto callbackMethod = getClass()->getMethod("callback"); auto result = callbackMethod(self(), frame, params); diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index 068bad3a7a..f6f7090f11 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -23,8 +23,8 @@ struct JFrameProcessorPlugin : public JavaClass { /** * Call the plugin. */ - local_ref callback(alias_ref frame, - alias_ref params) const; + local_ref callback(const alias_ref& frame, + const alias_ref& params) const; }; } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp index 5cecfd775b..9a5cbc1a96 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp @@ -5,6 +5,7 @@ #include "JVisionCameraProxy.h" #include +#include #include #include #include @@ -18,10 +19,10 @@ using TJSCallInvokerHolder = jni::alias_ref; using TOptions = jni::local_ref; -JVisionCameraProxy::JVisionCameraProxy(jni::alias_ref javaThis, +JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref& javaThis, jsi::Runtime* runtime, - std::shared_ptr callInvoker, - jni::global_ref scheduler) { + const std::shared_ptr& callInvoker, + const jni::global_ref& scheduler) { _javaPart = make_global(javaThis); __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); @@ -46,7 +47,7 @@ JVisionCameraProxy::JVisionCameraProxy(jni::alias_ref frameProcessor) { + const alias_ref& frameProcessor) { auto setFrameProcessorMethod = javaClassLocal()->getMethod)>("setFrameProcessor"); setFrameProcessorMethod(_javaPart, viewTag, frameProcessor); } @@ -56,10 +57,10 @@ void JVisionCameraProxy::removeFrameProcessor(int viewTag) { removeFrameProcessorMethod(_javaPart, viewTag); } -local_ref JVisionCameraProxy::getFrameProcessorPlugin(std::string name, +local_ref JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name, TOptions options) { - auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, TOptions)>("setFrameProcessor"); - return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), options); + auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, TOptions)>("getFrameProcessorPlugin"); + return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options)); } void JVisionCameraProxy::registerNatives() { @@ -72,7 +73,7 @@ TSelf JVisionCameraProxy::initHybrid( alias_ref jThis, jlong jsRuntimePointer, TJSCallInvokerHolder jsCallInvokerHolder, - TScheduler scheduler) { + const TScheduler& scheduler) { __android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy..."); // cast from JNI hybrid objects to C++ instances diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h index 0285b7141b..7aef2e19ce 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h @@ -22,9 +22,9 @@ class JVisionCameraProxy : public jni::HybridClass { static void registerNatives(); void setFrameProcessor(int viewTag, - jni::alias_ref frameProcessor); + const jni::alias_ref& frameProcessor); void removeFrameProcessor(int viewTag); - jni::local_ref getFrameProcessorPlugin(std::string name, + jni::local_ref getFrameProcessorPlugin(const std::string& name, jni::local_ref options); public: @@ -39,14 +39,14 @@ class JVisionCameraProxy : public jni::HybridClass { static auto constexpr TAG = "VisionCameraProxy"; static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;"; - explicit JVisionCameraProxy(jni::alias_ref javaThis, + explicit JVisionCameraProxy(const jni::alias_ref& javaThis, jsi::Runtime* jsRuntime, - std::shared_ptr jsCallInvoker, - jni::global_ref scheduler); + const std::shared_ptr& jsCallInvoker, + const jni::global_ref& scheduler); static jni::local_ref initHybrid(jni::alias_ref javaThis, jlong jsRuntimePointer, jni::alias_ref jsCallInvokerHolder, - jni::alias_ref scheduler); + const jni::alias_ref& scheduler); }; } diff --git a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp index 6e10502e66..7490643dc1 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp @@ -14,7 +14,7 @@ TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref jThis) { return makeCxxInstance(jThis); } -void JVisionCameraScheduler::dispatchAsync(std::function job) { +void JVisionCameraScheduler::dispatchAsync(const std::function& job) { // 1. add job to queue _jobs.push(job); scheduleTrigger(); diff --git a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h index ca4366e12b..34aaae863a 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h @@ -30,7 +30,7 @@ class JVisionCameraScheduler : public jni::HybridClass { static void registerNatives(); // schedules the given job to be run on the VisionCamera FP Thread at some future point in time - void dispatchAsync(std::function job); + void dispatchAsync(const std::function& job); private: friend HybridBase; diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index f8f85da3a0..d5ccb2edc6 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -15,21 +15,25 @@ public Frame(ImageProxy imageProxy) { this.imageProxy = imageProxy; } + public ImageProxy getImageProxy() { + return imageProxy; + } + @SuppressWarnings("unused") @DoNotStrip - private int getWidth() { + public int getWidth() { return imageProxy.getWidth(); } @SuppressWarnings("unused") @DoNotStrip - private int getHeight() { + public int getHeight() { return imageProxy.getHeight(); } @SuppressWarnings("unused") @DoNotStrip - private boolean getIsValid() { + public boolean getIsValid() { try { @SuppressLint("UnsafeOptInUsageError") Image image = imageProxy.getImage(); @@ -46,7 +50,7 @@ private boolean getIsValid() { @SuppressWarnings("unused") @DoNotStrip - private boolean getIsMirrored() { + public boolean getIsMirrored() { Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); // TODO: Figure out how to get isMirrored from ImageProxy return false; @@ -54,13 +58,13 @@ private boolean getIsMirrored() { @SuppressWarnings("unused") @DoNotStrip - private long getTimestamp() { + public long getTimestamp() { return imageProxy.getImageInfo().getTimestamp(); } @SuppressWarnings("unused") @DoNotStrip - private String getOrientation() { + public String getOrientation() { int rotation = imageProxy.getImageInfo().getRotationDegrees(); if (rotation >= 45 && rotation < 135) return "landscapeRight"; @@ -73,13 +77,13 @@ private String getOrientation() { @SuppressWarnings("unused") @DoNotStrip - private int getPlanesCount() { + public int getPlanesCount() { return imageProxy.getPlanes().length; } @SuppressWarnings("unused") @DoNotStrip - private int getBytesPerRow() { + public int getBytesPerRow() { return imageProxy.getPlanes()[0].getRowStride(); } @@ -87,7 +91,7 @@ private int getBytesPerRow() { @SuppressWarnings("unused") @DoNotStrip - private byte[] toByteArray() { + public byte[] toByteArray() { switch (imageProxy.getFormat()) { case ImageFormat.YUV_420_888: ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java index 5ed2adcca2..d3b631f6ef 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java @@ -3,7 +3,6 @@ import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import androidx.camera.core.ImageProxy; import com.facebook.proguard.annotations.DoNotStrip; import com.facebook.react.bridge.ReadableNativeMap; @@ -15,12 +14,12 @@ public abstract class FrameProcessorPlugin { /** * The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives. - * @param image The CameraX ImageProxy. Don't call .close() on this, as VisionCamera handles that. + * @param frame The Frame from the Camera. Don't call .close() on this, as VisionCamera handles that. * @return You can return any primitive, map or array you want. See the * Types * table for a list of supported types. */ @DoNotStrip @Keep - public abstract @Nullable Object callback(@NonNull ImageProxy image, @Nullable ReadableNativeMap params); + public abstract @Nullable Object callback(@NonNull Frame frame, @Nullable ReadableNativeMap params); } diff --git a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java index a33e7607e9..ff984a5cb9 100644 --- a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java +++ b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java @@ -1,11 +1,13 @@ package com.mrousavy.camera.example; import android.util.Log; + import androidx.camera.core.ImageProxy; import com.facebook.react.bridge.ReadableNativeMap; import com.facebook.react.bridge.WritableNativeArray; import com.facebook.react.bridge.WritableNativeMap; +import com.mrousavy.camera.frameprocessor.Frame; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -14,8 +16,9 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { @Override - public Object callback(@NotNull ImageProxy image, @Nullable ReadableNativeMap params) { + public Object callback(@NotNull Frame frame, @Nullable ReadableNativeMap params) { HashMap hashMap = params != null ? params.toHashMap() : new HashMap<>(); + ImageProxy image = frame.getImageProxy(); Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + hashMap.size() + " parameters:"); diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index e4e22882eb..710d523811 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -27,6 +27,7 @@ import type { NativeStackScreenProps } from '@react-navigation/native-stack'; import { useIsFocused } from '@react-navigation/core'; import { Skia } from '@shopify/react-native-skia'; import { FACE_SHADER } from './Shaders'; +import { examplePlugin } from './frame-processors/ExamplePlugin'; const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera); Reanimated.addWhitelistedNativeProps({ @@ -218,10 +219,12 @@ export function CameraPage({ navigation }: Props): React.ReactElement { const paint = Skia.Paint(); paint.setImageFilter(imageFilter); - const isIOS = Platform.OS === 'ios'; const frameProcessor = useFrameProcessor((frame) => { 'worklet'; + console.log(`Width: ${frame.width}`); + const result = examplePlugin(frame); + console.log('Example Plugin: ', result); }, []); return ( From c2885450bfd8d7a491ebd34f305b17b3bd6a59ea Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 23:53:08 +0200 Subject: [PATCH 06/11] Indents --- android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h | 2 +- android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt | 2 +- .../com/mrousavy/camera/frameprocessor/FrameProcessor.java | 2 +- example/src/CameraPage.tsx | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index f6f7090f11..a044e00551 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -19,7 +19,7 @@ using namespace jni; struct JFrameProcessorPlugin : public JavaClass { static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;"; - public: +public: /** * Call the plugin. */ diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt index c945c0b634..0a13fa3540 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt @@ -18,7 +18,7 @@ suspend fun CameraView.focus(pointMap: ReadableMap) { // Getting the point from the previewView needs to be run on the UI thread val point = withContext(coroutineScope.coroutineContext) { - previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat()); + previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat()) } val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE) diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java index c89cf7bcee..ec9501a960 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java @@ -19,7 +19,7 @@ public final class FrameProcessor { @DoNotStrip @Keep - private HybridData mHybridData; + private final HybridData mHybridData; public FrameProcessor(HybridData hybridData) { mHybridData = hybridData; diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index 710d523811..29cbbb2a0b 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import { useRef, useState, useMemo, useCallback } from 'react'; -import { Platform, StyleSheet, Text, View } from 'react-native'; +import { StyleSheet, Text, View } from 'react-native'; import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler'; import { CameraDeviceFormat, @@ -9,7 +9,6 @@ import { sortFormats, useCameraDevices, useFrameProcessor, - useSkiaFrameProcessor, VideoFile, } from 'react-native-vision-camera'; import { Camera, frameRateIncluded } from 'react-native-vision-camera'; From 456571f0abb04a569d16a5202d201390acb2499a Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 23:53:39 +0200 Subject: [PATCH 07/11] Cleanup --- .../main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt | 2 +- android/src/main/java/com/mrousavy/camera/Errors.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt index 3f3299bf2f..bb40925412 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt @@ -14,7 +14,7 @@ import java.io.FileOutputStream import kotlinx.coroutines.guava.await suspend fun CameraView.takeSnapshot(options: ReadableMap): WritableMap = coroutineScope { - val camera = camera ?: throw com.mrousavy.camera.CameraNotReadyError() + val camera = camera ?: throw CameraNotReadyError() val enableFlash = options.getString("flash") == "on" try { diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 5281222a06..2311bfbb83 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -43,12 +43,12 @@ class ParallelVideoProcessingNotSupportedError(cause: Throwable) : CameraError(" "See https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#the-supportsparallelvideoprocessing-prop for more information.", cause) class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given FPS were not valid for the currently selected format. Make sure you select a format which `frameRateRanges` includes $fps FPS!") -class HdrNotContainedInFormatError() : CameraError( +class HdrNotContainedInFormatError : CameraError( "format", "invalid-hdr", "The currently selected format does not support HDR capture! " + "Make sure you select a format which `frameRateRanges` includes `supportsPhotoHDR`!" ) -class LowLightBoostNotContainedInFormatError() : CameraError( +class LowLightBoostNotContainedInFormatError : CameraError( "format", "invalid-low-light-boost", "The currently selected format does not support low-light boost (night mode)! " + "Make sure you select a format which includes `supportsLowLightBoost`." From 8b11200939cdf6f66b9fe9d921704b6a3321b297 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 23:54:16 +0200 Subject: [PATCH 08/11] indents --- android/src/main/cpp/FrameHostObject.h | 8 ++++---- .../src/main/cpp/java-bindings/JVisionCameraScheduler.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/android/src/main/cpp/FrameHostObject.h b/android/src/main/cpp/FrameHostObject.h index fc4f5214a2..279c01eac3 100644 --- a/android/src/main/cpp/FrameHostObject.h +++ b/android/src/main/cpp/FrameHostObject.h @@ -18,18 +18,18 @@ namespace vision { using namespace facebook; class JSI_EXPORT FrameHostObject : public jsi::HostObject { - public: explicit FrameHostObject(const jni::alias_ref& frame); +public: ~FrameHostObject(); - public: +public: jsi::Value get(jsi::Runtime &, const jsi::PropNameID &name) override; std::vector getPropertyNames(jsi::Runtime &rt) override; - public: +public: jni::global_ref frame; - private: +private: static auto constexpr TAG = "VisionCamera"; size_t _refCount; diff --git a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h index 34aaae863a..f70ab5c9a7 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h @@ -24,7 +24,7 @@ using namespace facebook; * 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1. */ class JVisionCameraScheduler : public jni::HybridClass { - public: +public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;"; static jni::local_ref initHybrid(jni::alias_ref jThis); static void registerNatives(); @@ -32,7 +32,7 @@ class JVisionCameraScheduler : public jni::HybridClass { // schedules the given job to be run on the VisionCamera FP Thread at some future point in time void dispatchAsync(const std::function& job); - private: +private: friend HybridBase; jni::global_ref javaPart_; std::queue> _jobs; From cdfb713dd6da4e15af7b28323c5012334d748542 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 22 Jul 2023 00:00:18 +0200 Subject: [PATCH 09/11] docs: Update Android docs --- .../java/com/mrousavy/camera/CameraView.kt | 6 +++++ .../FRAME_PROCESSORS_CREATE_OVERVIEW.mdx | 6 +++-- .../FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx | 24 +++++++------------ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 60116b17e4..cbe53eefe5 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -26,6 +26,8 @@ import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.* import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin +import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlinx.coroutines.guava.await @@ -174,6 +176,10 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer return true } else { if (video == true && enableFrameProcessor) { + FrameProcessorPluginRegistry.addFrameProcessorPlugin("") { options -> + FrameProcessorPlugin() + } + // Camera supports max. 2 use-cases, but both are occupied by `frameProcessor` and `video` return true } else { diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index e94f8b7246..13fe39d589 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -131,7 +131,7 @@ const frameProcessor = useFrameProcessor((frame) => { ## What's possible? -You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame (`CMSampleBuffer` on iOS, `ImageProxy` on Android) which you can use however you want. In other words; **everything is possible**. +You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame ([`CMSampleBuffer`][5] on iOS, [`ImageProxy`][6] on Android) which you can use however you want. In other words; **everything is possible**. ## Implementations @@ -194,5 +194,7 @@ Your Frame Processor Plugins have to be fast. Use the FPS Graph (`enableFpsGraph [1]: https://github.com/mrousavy/react-native-vision-camera/blob/main/src/Frame.ts [2]: https://github.com/mrousavy/react-native-vision-camera/blob/main/ios/Frame%20Processor/Frame.h -[3]: https://developer.android.com/reference/androidx/camera/core/ImageProxy +[3]: https://github.com/mrousavy/react-native-vision-camera/blob/main/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java [4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24 +[5]: https://developer.apple.com/documentation/coremedia/cmsamplebuffer +[6]: https://developer.android.com/reference/androidx/camera/core/ImageProxy diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx index d6667bfd49..6de0debda0 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx @@ -55,21 +55,16 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera- 3. Add the following code: ```java {8} -import androidx.camera.core.ImageProxy; +import com.mrousavy.camera.frameprocessor.Frame; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin { @Override - public Object callback(ImageProxy image, ReadableNativeMap arguments) { + public Object callback(Frame frame, ReadableNativeMap arguments) { // code goes here return null; } - - @Override - public String getName() { - return "detectFaces"; - } } ``` @@ -86,13 +81,14 @@ import com.facebook.react.bridge.NativeModule; import com.facebook.react.bridge.ReactApplicationContext; import com.facebook.react.uimanager.ViewManager; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; +import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry; import javax.annotation.Nonnull; public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { @NonNull @Override public List createNativeModules(@NonNull ReactApplicationContext reactContext) { - FrameProcessorPlugin.register(new FaceDetectorFrameProcessorPlugin()); + FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin()); return Collections.emptyList(); } @@ -125,19 +121,15 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { 3. Add the following code: ```kotlin {7} -import androidx.camera.core.ImageProxy +import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() { - override fun callback(image: ImageProxy, arguments: ReadableNativeMap): Any? { + override fun callback(frame: Frame, arguments: ReadableNativeMap): Any? { // code goes here return null } - - override fun getName(): String { - return "detectFaces" - } } ``` @@ -157,7 +149,9 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin class FaceDetectorFrameProcessorPluginPackage : ReactPackage { override fun createNativeModules(reactContext: ReactApplicationContext): List { - FrameProcessorPlugin.register(FaceDetectorFrameProcessorPlugin()) + FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options -> + FaceDetectorFrameProcessorPlugin() + } return emptyList() } From ca4bf22858b1a2c26db84ec98cdbfc812e95e7b5 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 22 Jul 2023 00:00:28 +0200 Subject: [PATCH 10/11] Update CameraView.kt --- android/src/main/java/com/mrousavy/camera/CameraView.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index cbe53eefe5..5cc7654c77 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -176,10 +176,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer return true } else { if (video == true && enableFrameProcessor) { - FrameProcessorPluginRegistry.addFrameProcessorPlugin("") { options -> - FrameProcessorPlugin() - } - // Camera supports max. 2 use-cases, but both are occupied by `frameProcessor` and `video` return true } else { From f8aec5519264ce1142ca8c6444a99dcd12501247 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Sat, 22 Jul 2023 00:12:21 +0200 Subject: [PATCH 11/11] fix: Format C++ code --- android/src/main/cpp/FrameHostObject.h | 8 +- .../cpp/FrameProcessorPluginHostObject.cpp | 3 +- .../main/cpp/FrameProcessorPluginHostObject.h | 19 ++-- android/src/main/cpp/VisionCameraProxy.cpp | 14 ++- android/src/main/cpp/VisionCameraProxy.h | 15 +-- .../main/cpp/java-bindings/JFrameProcessor.h | 10 +- .../cpp/java-bindings/JFrameProcessorPlugin.h | 2 +- .../cpp/java-bindings/JVisionCameraProxy.cpp | 98 ++++++++++--------- .../cpp/java-bindings/JVisionCameraProxy.h | 63 ++++++------ .../java-bindings/JVisionCameraScheduler.h | 4 +- 10 files changed, 126 insertions(+), 110 deletions(-) diff --git a/android/src/main/cpp/FrameHostObject.h b/android/src/main/cpp/FrameHostObject.h index 279c01eac3..92e5ff3ad9 100644 --- a/android/src/main/cpp/FrameHostObject.h +++ b/android/src/main/cpp/FrameHostObject.h @@ -19,17 +19,17 @@ using namespace facebook; class JSI_EXPORT FrameHostObject : public jsi::HostObject { explicit FrameHostObject(const jni::alias_ref& frame); -public: + public: ~FrameHostObject(); -public: + public: jsi::Value get(jsi::Runtime &, const jsi::PropNameID &name) override; std::vector getPropertyNames(jsi::Runtime &rt) override; -public: + public: jni::global_ref frame; -private: + private: static auto constexpr TAG = "VisionCamera"; size_t _refCount; diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.cpp b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp index fce813d062..43312d3ef3 100644 --- a/android/src/main/cpp/FrameProcessorPluginHostObject.cpp +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp @@ -6,6 +6,7 @@ #include #include "FrameHostObject.h" #include "JSIJNIConversion.h" +#include namespace vision { @@ -49,4 +50,4 @@ jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi: return jsi::Value::undefined(); } -} // namespace vision \ No newline at end of file +} // namespace vision diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.h b/android/src/main/cpp/FrameProcessorPluginHostObject.h index 1645dd715f..20a516e09c 100644 --- a/android/src/main/cpp/FrameProcessorPluginHostObject.h +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.h @@ -9,23 +9,24 @@ #include #include #include +#include namespace vision { using namespace facebook; class FrameProcessorPluginHostObject: public jsi::HostObject { -public: - explicit FrameProcessorPluginHostObject(jni::alias_ref plugin): + public: + explicit FrameProcessorPluginHostObject(jni::alias_ref plugin): _plugin(make_global(plugin)) { } - ~FrameProcessorPluginHostObject() { } + ~FrameProcessorPluginHostObject() { } -public: - std::vector getPropertyNames(jsi::Runtime& runtime) override; - jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; + public: + std::vector getPropertyNames(jsi::Runtime& runtime) override; + jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; -private: - jni::global_ref _plugin; + private: + jni::global_ref _plugin; }; -} // namespace vision \ No newline at end of file +} // namespace vision diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp index 1ffdf31e44..ba302cdf8e 100644 --- a/android/src/main/cpp/VisionCameraProxy.cpp +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -15,6 +15,10 @@ #include "JSITypedArray.h" #include "FrameProcessorPluginHostObject.h" +#include +#include +#include + namespace vision { using namespace facebook; @@ -26,7 +30,7 @@ VisionCameraProxy::VisionCameraProxy(const jni::alias_refcthis()->getWorkletContext(); + auto workletContext = _javaProxy->cthis()->getWorkletContext(); invalidateArrayBufferCache(*workletContext->getJsRuntime()); invalidateArrayBufferCache(workletContext->getWorkletRuntime()); } @@ -58,11 +62,11 @@ void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, co throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType); } - _javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor)); + _javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor)); } void VisionCameraProxy::removeFrameProcessor(int viewTag) { - _javaProxy->cthis()->removeFrameProcessor(viewTag); + _javaProxy->cthis()->removeFrameProcessor(viewTag); } jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, @@ -136,7 +140,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& void VisionCameraInstaller::install(jni::alias_ref, - jni::alias_ref proxy) { + jni::alias_ref proxy) { // global.VisionCameraProxy auto visionCameraProxy = std::make_shared(proxy); jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime(); @@ -145,4 +149,4 @@ void VisionCameraInstaller::install(jni::alias_ref, jsi::Object::createFromHostObject(runtime, visionCameraProxy)); } -} \ No newline at end of file +} // namespace vision diff --git a/android/src/main/cpp/VisionCameraProxy.h b/android/src/main/cpp/VisionCameraProxy.h index fe001c6105..65830b1524 100644 --- a/android/src/main/cpp/VisionCameraProxy.h +++ b/android/src/main/cpp/VisionCameraProxy.h @@ -9,32 +9,35 @@ #include "java-bindings/JVisionCameraScheduler.h" #include "java-bindings/JVisionCameraProxy.h" +#include +#include + namespace vision { using namespace facebook; class VisionCameraProxy: public jsi::HostObject { -public: + public: explicit VisionCameraProxy(const jni::alias_ref& javaProxy); ~VisionCameraProxy(); -public: + public: std::vector getPropertyNames(jsi::Runtime& runtime) override; jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; -private: + private: void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor); void removeFrameProcessor(int viewTag); jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options); -private: + private: jni::global_ref _javaProxy; static constexpr const char* TAG = "VisionCameraProxy"; }; class VisionCameraInstaller: public jni::JavaClass { -public: + public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;"; static void registerNatives() { javaClassStatic()->registerNatives({ @@ -45,4 +48,4 @@ class VisionCameraInstaller: public jni::JavaClass { jni::alias_ref proxy); }; -} \ No newline at end of file +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.h b/android/src/main/cpp/java-bindings/JFrameProcessor.h index b6c1bad108..063b9fdd3b 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessor.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.h @@ -20,27 +20,27 @@ namespace vision { using namespace facebook; struct JFrameProcessor : public jni::HybridClass { -public: + public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;"; static void registerNatives(); static jni::local_ref create(const std::shared_ptr& worklet, const std::shared_ptr& context); -public: + public: /** * Call the JS Frame Processor. */ void call(alias_ref frame); -private: + private: // Private constructor. Use `create(..)` to create new instances. explicit JFrameProcessor(std::shared_ptr worklet, std::shared_ptr context); -private: + private: void callWithFrameHostObject(const std::shared_ptr& frameHostObject) const; -private: + private: friend HybridBase; std::shared_ptr _workletInvoker; std::shared_ptr _workletContext; diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index a044e00551..f6f7090f11 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -19,7 +19,7 @@ using namespace jni; struct JFrameProcessorPlugin : public JavaClass { static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;"; -public: + public: /** * Call the plugin. */ diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp index 9a5cbc1a96..73c1cd0dd7 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp @@ -6,11 +6,15 @@ #include #include +#include + #include +#include + #include #include + #include "FrameProcessorPluginHostObject.h" -#include namespace vision { @@ -20,68 +24,68 @@ using TScheduler = jni::alias_ref; using TOptions = jni::local_ref; JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref& javaThis, - jsi::Runtime* runtime, - const std::shared_ptr& callInvoker, - const jni::global_ref& scheduler) { - _javaPart = make_global(javaThis); - - __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); - - auto runOnJS = [callInvoker](std::function&& f) { - // Run on React JS Runtime - callInvoker->invokeAsync(std::move(f)); - }; - auto runOnWorklet = [scheduler](std::function&& f) { - // Run on Frame Processor Worklet Runtime - scheduler->cthis()->dispatchAsync([f = std::move(f)](){ - f(); - }); - }; - _workletContext = std::make_shared("VisionCamera", - runtime, - runOnJS, - runOnWorklet); - __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!"); + jsi::Runtime* runtime, + const std::shared_ptr& callInvoker, + const jni::global_ref& scheduler) { + _javaPart = make_global(javaThis); + + __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); + + auto runOnJS = [callInvoker](std::function&& f) { + // Run on React JS Runtime + callInvoker->invokeAsync(std::move(f)); + }; + auto runOnWorklet = [scheduler](std::function&& f) { + // Run on Frame Processor Worklet Runtime + scheduler->cthis()->dispatchAsync([f = std::move(f)](){ + f(); + }); + }; + _workletContext = std::make_shared("VisionCamera", + runtime, + runOnJS, + runOnWorklet); + __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!"); } void JVisionCameraProxy::setFrameProcessor(int viewTag, - const alias_ref& frameProcessor) { - auto setFrameProcessorMethod = javaClassLocal()->getMethod)>("setFrameProcessor"); - setFrameProcessorMethod(_javaPart, viewTag, frameProcessor); + const alias_ref& frameProcessor) { + auto setFrameProcessorMethod = javaClassLocal()->getMethod)>("setFrameProcessor"); + setFrameProcessorMethod(_javaPart, viewTag, frameProcessor); } void JVisionCameraProxy::removeFrameProcessor(int viewTag) { - auto removeFrameProcessorMethod = javaClassLocal()->getMethod("removeFrameProcessor"); - removeFrameProcessorMethod(_javaPart, viewTag); + auto removeFrameProcessorMethod = javaClassLocal()->getMethod("removeFrameProcessor"); + removeFrameProcessorMethod(_javaPart, viewTag); } local_ref JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name, - TOptions options) { - auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, TOptions)>("getFrameProcessorPlugin"); - return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options)); + TOptions options) { + auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, TOptions)>("getFrameProcessorPlugin"); + return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options)); } void JVisionCameraProxy::registerNatives() { - registerHybrid({ - makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid) - }); + registerHybrid({ + makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid) + }); } TSelf JVisionCameraProxy::initHybrid( - alias_ref jThis, - jlong jsRuntimePointer, - TJSCallInvokerHolder jsCallInvokerHolder, - const TScheduler& scheduler) { - __android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy..."); - - // cast from JNI hybrid objects to C++ instances - auto jsRuntime = reinterpret_cast(jsRuntimePointer); - auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); - auto sharedScheduler = make_global(scheduler); - - return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler); + alias_ref jThis, + jlong jsRuntimePointer, + TJSCallInvokerHolder jsCallInvokerHolder, + const TScheduler& scheduler) { + __android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy..."); + + // cast from JNI hybrid objects to C++ instances + auto jsRuntime = reinterpret_cast(jsRuntimePointer); + auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); + auto sharedScheduler = make_global(scheduler); + + return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler); } -} \ No newline at end of file +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h index 7aef2e19ce..29c0ab55db 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h @@ -13,40 +13,43 @@ #include "JVisionCameraScheduler.h" #include "JFrameProcessor.h" +#include +#include + namespace vision { using namespace facebook; class JVisionCameraProxy : public jni::HybridClass { -public: - static void registerNatives(); - - void setFrameProcessor(int viewTag, - const jni::alias_ref& frameProcessor); - void removeFrameProcessor(int viewTag); - jni::local_ref getFrameProcessorPlugin(const std::string& name, - jni::local_ref options); - -public: - std::shared_ptr getWorkletContext() { return _workletContext; } - -private: - std::shared_ptr _workletContext; - -private: - friend HybridBase; - jni::global_ref _javaPart; - static auto constexpr TAG = "VisionCameraProxy"; - static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;"; - - explicit JVisionCameraProxy(const jni::alias_ref& javaThis, - jsi::Runtime* jsRuntime, - const std::shared_ptr& jsCallInvoker, - const jni::global_ref& scheduler); - static jni::local_ref initHybrid(jni::alias_ref javaThis, - jlong jsRuntimePointer, - jni::alias_ref jsCallInvokerHolder, - const jni::alias_ref& scheduler); + public: + static void registerNatives(); + + void setFrameProcessor(int viewTag, + const jni::alias_ref& frameProcessor); + void removeFrameProcessor(int viewTag); + jni::local_ref getFrameProcessorPlugin(const std::string& name, + jni::local_ref options); + + public: + std::shared_ptr getWorkletContext() { return _workletContext; } + + private: + std::shared_ptr _workletContext; + + private: + friend HybridBase; + jni::global_ref _javaPart; + static auto constexpr TAG = "VisionCameraProxy"; + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;"; + + explicit JVisionCameraProxy(const jni::alias_ref& javaThis, + jsi::Runtime* jsRuntime, + const std::shared_ptr& jsCallInvoker, + const jni::global_ref& scheduler); + static jni::local_ref initHybrid(jni::alias_ref javaThis, + jlong jsRuntimePointer, + jni::alias_ref jsCallInvokerHolder, + const jni::alias_ref& scheduler); }; -} +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h index f70ab5c9a7..34aaae863a 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h @@ -24,7 +24,7 @@ using namespace facebook; * 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1. */ class JVisionCameraScheduler : public jni::HybridClass { -public: + public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;"; static jni::local_ref initHybrid(jni::alias_ref jThis); static void registerNatives(); @@ -32,7 +32,7 @@ class JVisionCameraScheduler : public jni::HybridClass { // schedules the given job to be run on the VisionCamera FP Thread at some future point in time void dispatchAsync(const std::function& job); -private: + private: friend HybridBase; jni::global_ref javaPart_; std::queue> _jobs;