From 0611b736ccdbad381eb0c6ddee23e356aed9cacf Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 21 Jul 2023 17:51:49 +0200 Subject: [PATCH] First Android rewrite --- android/CMakeLists.txt | 14 +- android/src/main/cpp/FrameHostObject.cpp | 2 +- android/src/main/cpp/FrameHostObject.h | 6 +- .../cpp/FrameProcessorPluginHostObject.cpp | 52 ++++++ .../main/cpp/FrameProcessorPluginHostObject.h | 31 ++++ android/src/main/cpp/JSIJNIConversion.cpp | 9 +- android/src/main/cpp/VisionCamera.cpp | 11 +- android/src/main/cpp/VisionCameraProxy.cpp | 172 ++++++++++++++++++ android/src/main/cpp/VisionCameraProxy.h | 60 ++++++ .../JCameraView.cpp} | 18 +- .../JCameraView.h} | 6 +- android/src/main/cpp/java-bindings/JFrame.cpp | 65 +++++++ .../java-bindings/{JImageProxy.h => JFrame.h} | 6 +- .../cpp/java-bindings/JFrameProcessor.cpp | 28 +++ .../main/cpp/java-bindings/JFrameProcessor.h | 40 ++++ .../java-bindings/JFrameProcessorPlugin.cpp | 8 +- .../cpp/java-bindings/JFrameProcessorPlugin.h | 6 +- .../main/cpp/java-bindings/JImageProxy.cpp | 78 -------- .../JVisionCameraScheduler.cpp} | 18 +- .../JVisionCameraScheduler.h} | 6 +- .../java/com/mrousavy/camera/CameraView.kt | 3 - .../com/mrousavy/camera/CameraViewModule.kt | 56 ++---- .../mrousavy/camera/frameprocessor/Frame.java | 130 +++++++++++++ .../camera/frameprocessor/FrameProcessor.java | 16 ++ .../frameprocessor/FrameProcessorPlugin.java | 2 +- .../frameprocessor/ImageProxyUtils.java | 98 ---------- .../frameprocessor/VisionCameraInstaller.java | 9 + ...RuntimeManager.kt => VisionCameraProxy.kt} | 9 +- .../camera/parsers/PermissionStatus+String.kt | 11 ++ .../FrameProcessorPluginHostObject.mm | 2 + ios/Frame Processor/VisionCameraProxy.mm | 2 +- 31 files changed, 693 insertions(+), 281 deletions(-) create mode 100644 android/src/main/cpp/FrameProcessorPluginHostObject.cpp create mode 100644 android/src/main/cpp/FrameProcessorPluginHostObject.h create mode 100644 android/src/main/cpp/VisionCameraProxy.cpp create mode 100644 android/src/main/cpp/VisionCameraProxy.h rename android/src/main/cpp/{CameraView.cpp => java-bindings/JCameraView.cpp} (66%) rename android/src/main/cpp/{CameraView.h => java-bindings/JCameraView.h} (83%) create mode 100644 android/src/main/cpp/java-bindings/JFrame.cpp rename android/src/main/cpp/java-bindings/{JImageProxy.h => JFrame.h} (73%) create mode 100644 android/src/main/cpp/java-bindings/JFrameProcessor.cpp create mode 100644 android/src/main/cpp/java-bindings/JFrameProcessor.h delete mode 100644 android/src/main/cpp/java-bindings/JImageProxy.cpp rename android/src/main/cpp/{VisionCameraScheduler.cpp => java-bindings/JVisionCameraScheduler.cpp} (52%) rename android/src/main/cpp/{VisionCameraScheduler.h => java-bindings/JVisionCameraScheduler.h} (85%) create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java delete mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java rename android/src/main/java/com/mrousavy/camera/frameprocessor/{FrameProcessorRuntimeManager.kt => VisionCameraProxy.kt} (88%) create mode 100644 android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 088f664c98..12e37cf902 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -32,15 +32,17 @@ add_library( ${PACKAGE_NAME} SHARED ../cpp/JSITypedArray.cpp - src/main/cpp/VisionCamera.cpp - src/main/cpp/JSIJNIConversion.cpp src/main/cpp/FrameHostObject.cpp - src/main/cpp/FrameProcessorRuntimeManager.cpp - src/main/cpp/CameraView.cpp - src/main/cpp/VisionCameraScheduler.cpp + src/main/cpp/FrameProcessorPluginHostObject.cpp + src/main/cpp/JSIJNIConversion.cpp + src/main/cpp/VisionCamera.cpp + src/main/cpp/VisionCameraProxy.cpp + src/main/cpp/java-bindings/JCameraView.cpp + src/main/cpp/java-bindings/JFrame.cpp + src/main/cpp/java-bindings/JFrameProcessor.cpp src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp - src/main/cpp/java-bindings/JImageProxy.cpp src/main/cpp/java-bindings/JHashMap.cpp + src/main/cpp/java-bindings/JVisionCameraScheduler.cpp ) # Header Search Paths (includes) diff --git a/android/src/main/cpp/FrameHostObject.cpp b/android/src/main/cpp/FrameHostObject.cpp index 60b367ae12..cb753cf6d8 100644 --- a/android/src/main/cpp/FrameHostObject.cpp +++ b/android/src/main/cpp/FrameHostObject.cpp @@ -18,7 +18,7 @@ namespace vision { using namespace facebook; -FrameHostObject::FrameHostObject(jni::alias_ref image): frame(make_global(image)), _refCount(0) { } +FrameHostObject::FrameHostObject(jni::alias_ref frame): frame(make_global(frame)), _refCount(0) { } FrameHostObject::~FrameHostObject() { // Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread diff --git a/android/src/main/cpp/FrameHostObject.h b/android/src/main/cpp/FrameHostObject.h index eb22db52ac..53d045b117 100644 --- a/android/src/main/cpp/FrameHostObject.h +++ b/android/src/main/cpp/FrameHostObject.h @@ -11,7 +11,7 @@ #include #include -#include "java-bindings/JImageProxy.h" +#include "java-bindings/JFrame.h" namespace vision { @@ -19,7 +19,7 @@ using namespace facebook; class JSI_EXPORT FrameHostObject : public jsi::HostObject { public: - explicit FrameHostObject(jni::alias_ref image); + explicit FrameHostObject(jni::alias_ref frame); ~FrameHostObject(); public: @@ -27,7 +27,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject { std::vector getPropertyNames(jsi::Runtime &rt) override; public: - jni::global_ref frame; + jni::global_ref frame; private: static auto constexpr TAG = "VisionCamera"; diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.cpp b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp new file mode 100644 index 0000000000..f219ca95f0 --- /dev/null +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp @@ -0,0 +1,52 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "FrameProcessorPluginHostObject.h" +#include +#include "FrameHostObject.h" +#include "JSIJNIConversion.h" + +namespace vision { + +using namespace facebook; + +std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime &runtime) { + std::vector result; + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); + return result; +} + +jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime &runtime, const jsi::PropNameID &propName) { + auto name = propName.utf8(runtime); + + if (name == "call") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "call"), + 2, + [=](jsi::Runtime &runtime, + const jsi::Value &thisValue, + const jsi::Value *arguments, + size_t count) -> jsi::Value { + // Frame is first argument + auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime); + auto frame = frameHostObject->frame; + + // Options are second argument (possibly undefined) + jobject options = nullptr; + if (count > 1) { + options = JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[1]); + } + + // Call actual plugin + auto result = _plugin->callback(frame, options); + + // Convert result value to jsi::Value (possibly undefined) + return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result); + }); + } + + return jsi::Value::undefined(); +} + +} // namespace vision \ No newline at end of file diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.h b/android/src/main/cpp/FrameProcessorPluginHostObject.h new file mode 100644 index 0000000000..944becc68e --- /dev/null +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.h @@ -0,0 +1,31 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include +#include "java-bindings/JFrameProcessorPlugin.h" +#include +#include +#include + +namespace vision { + +using namespace facebook; + +class FrameProcessorPluginHostObject: public jsi::HostObject { +public: + explicit FrameProcessorPluginHostObject(jni::global_ref plugin): + _plugin(plugin) { } + ~FrameProcessorPluginHostObject() { } + +public: + std::vector getPropertyNames(jsi::Runtime& runtime) override; + jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; + +private: + jni::global_ref _plugin; +}; + +} // namespace vision \ No newline at end of file diff --git a/android/src/main/cpp/JSIJNIConversion.cpp b/android/src/main/cpp/JSIJNIConversion.cpp index d34308f6a2..879daf19fd 100644 --- a/android/src/main/cpp/JSIJNIConversion.cpp +++ b/android/src/main/cpp/JSIJNIConversion.cpp @@ -21,7 +21,7 @@ #include #include "FrameHostObject.h" -#include "java-bindings/JImageProxy.h" +#include "java-bindings/JFrame.h" #include "java-bindings/JArrayList.h" #include "java-bindings/JHashMap.h" @@ -178,10 +178,9 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c auto hashMap = toHashMapFunc(object.get()); return convertJNIObjectToJSIValue(runtime, hashMap); - } else if (object->isInstanceOf(JImageProxy::javaClassStatic())) { - // ImageProxy - - auto frame = static_ref_cast(object); + } else if (object->isInstanceOf(JFrame::javaClassStatic())) { + // Frame + auto frame = static_ref_cast(object); // box into HostObject auto hostObject = std::make_shared(frame); diff --git a/android/src/main/cpp/VisionCamera.cpp b/android/src/main/cpp/VisionCamera.cpp index 6b91b8ef25..5dc0eca03e 100644 --- a/android/src/main/cpp/VisionCamera.cpp +++ b/android/src/main/cpp/VisionCamera.cpp @@ -1,13 +1,14 @@ #include #include #include "FrameProcessorRuntimeManager.h" -#include "CameraView.h" -#include "VisionCameraScheduler.h" +#include "java-bindings/JCameraView.h" +#include "java-bindings/JVisionCameraScheduler.h" +#include "VisionCameraProxy.h" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { return facebook::jni::initialize(vm, [] { - vision::FrameProcessorRuntimeManager::registerNatives(); - vision::CameraView::registerNatives(); - vision::VisionCameraScheduler::registerNatives(); + vision::VisionCameraInstaller::registerNatives(); + vision::JCameraView::registerNatives(); + vision::JVisionCameraScheduler::registerNatives(); }); } diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp new file mode 100644 index 0000000000..c574de12a9 --- /dev/null +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -0,0 +1,172 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "VisionCameraProxy.h" +#include + +#include +#include + +#include "java-bindings/JCameraView.h" +#include "java-bindings/JFrameProcessor.h" +#include "java-bindings/JFrameProcessorPlugin.h" +#include "FrameProcessorPluginHostObject.h" +#include "FrameHostObject.h" +#include "JSIJNIConversion.h" +#include "JSITypedArray.h" + +#include + +namespace vision { + +using namespace facebook; + +VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime, + std::shared_ptr callInvoker, + jni::global_ref scheduler) { + _callInvoker = callInvoker; + _scheduler = scheduler; + + __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); + + auto runOnJS = [callInvoker](std::function&& f) { + // Run on React JS Runtime + callInvoker->invokeAsync(std::move(f)); + }; + auto runOnWorklet = [this](std::function&& f) { + // Run on Frame Processor Worklet Runtime + _scheduler->cthis()->dispatchAsync([f = std::move(f)](){ + f(); + }); + }; + _workletContext = std::make_shared("VisionCamera", + &runtime, + runOnJS, + runOnWorklet); + __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!"); +} + +VisionCameraProxy::~VisionCameraProxy() { + __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context..."); + // Destroy ArrayBuffer cache for both the JS and the Worklet Runtime. + vision::invalidateArrayBufferCache(*_workletContext->getJsRuntime()); + vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime()); +} + +std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) { + std::vector result; + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled"))); + return result; +} + +jni::global_ref VisionCameraProxy::findCameraViewById(int viewId) { + // TODO: implement findCameraViewById + /*static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod("findCameraViewById"); + auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId); + return make_global(weakCameraView);*/ + return nullptr; +} + +void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& object) { + auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime); + auto worklet = std::make_shared(runtime, object.getProperty(runtime, "frameProcessor")); + + auto view = findCameraViewById(viewTag); + JFrameProcessor frameProcessor(worklet, _workletContext); + + // TODO: Set frame processor on JCameraView +} + +void VisionCameraProxy::removeFrameProcessor(jsi::Runtime& runtime, int viewTag) { + auto view = findCameraViewById(viewTag); + + // TODO: Remove frame processor from JCameraView +} + +jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options) { + // TODO: Get Frame Processor Plugin here + + auto pluginHostObject = std::make_shared(plugin, _callInvoker); + return jsi::Object::createFromHostObject(runtime, pluginHostObject); +} + +jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { + auto name = propName.utf8(runtime); + + if (name == "isSkiaEnabled") { +#ifdef VISION_CAMERA_ENABLE_SKIA + return jsi::Value(true); +#else + return jsi::Value(false); +#endif + } + if (name == "setFrameProcessor") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + auto viewTag = arguments[0].asNumber(); + auto object = arguments[1].asObject(runtime); + this->setFrameProcessor(runtime, static_cast(viewTag), object); + return jsi::Value::undefined(); + }); + } + if (name == "removeFrameProcessor") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + auto viewTag = arguments[0].asNumber(); + this->removeFrameProcessor(runtime, static_cast(viewTag)); + return jsi::Value::undefined(); + }); + } + if (name == "getFrameProcessorPlugin") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + if (count < 1 || !arguments[0].isString()) { + throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); + } + auto pluginName = arguments[0].asString(runtime).utf8(runtime); + auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime); + + return this->getFrameProcessorPlugin(runtime, pluginName, options); + }); + } + + return jsi::Value::undefined(); +} + + +void VisionCameraInstaller::install(jni::alias_ref, + jlong jsiRuntimePtr, + jni::alias_ref callInvokerHolder, + jni::alias_ref scheduler) { + // cast from JNI hybrid objects to C++ instances + jsi::Runtime& runtime = *reinterpret_cast(jsiRuntimePtr); + std::shared_ptr callInvoker = callInvokerHolder->cthis()->getCallInvoker(); + jni::global_ref sharedScheduler = make_global(scheduler); + + // global.VisionCameraProxy + auto visionCameraProxy = std::make_shared(runtime, callInvoker, sharedScheduler); + runtime.global().setProperty(runtime, + "VisionCameraProxy", + jsi::Object::createFromHostObject(runtime, visionCameraProxy)); +} + +} \ No newline at end of file diff --git a/android/src/main/cpp/VisionCameraProxy.h b/android/src/main/cpp/VisionCameraProxy.h new file mode 100644 index 0000000000..b95e5440b8 --- /dev/null +++ b/android/src/main/cpp/VisionCameraProxy.h @@ -0,0 +1,60 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "java-bindings/JVisionCameraScheduler.h" +#include "java-bindings/JCameraView.h" + +namespace vision { + +using namespace facebook; + +class VisionCameraProxy: public jsi::HostObject { +public: + explicit VisionCameraProxy(jsi::Runtime& runtime, + std::shared_ptr callInvoker, + jni::global_ref scheduler); + ~VisionCameraProxy(); + +public: + std::vector getPropertyNames(jsi::Runtime& runtime) override; + jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; + +private: + void setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& frameProcessor); + void removeFrameProcessor(jsi::Runtime& runtime, int viewTag); + jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options); + jni::global_ref findCameraViewById(int viewId); + +private: + std::shared_ptr _workletContext; + std::shared_ptr _callInvoker; + jni::global_ref _scheduler; + static constexpr const char* TAG = "VisionCameraProxy"; +}; + + +class VisionCameraInstaller: public jni::JavaClass { +public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;"; + static void registerNatives() { + javaClassStatic()->registerNatives({ + makeNativeMethod("install", VisionCameraInstaller::install) + }); + } + static void install(jni::alias_ref clazz, + jlong jsiRuntimePtr, + jni::alias_ref callInvoker, + jni::alias_ref scheduler); +}; + +} \ No newline at end of file diff --git a/android/src/main/cpp/CameraView.cpp b/android/src/main/cpp/java-bindings/JCameraView.cpp similarity index 66% rename from android/src/main/cpp/CameraView.cpp rename to android/src/main/cpp/java-bindings/JCameraView.cpp index 665bd312b5..8ef63734eb 100644 --- a/android/src/main/cpp/CameraView.cpp +++ b/android/src/main/cpp/java-bindings/JCameraView.cpp @@ -2,7 +2,7 @@ // Created by Marc Rousavy on 14.06.21. // -#include "CameraView.h" +#include "JCameraView.h" #include #include @@ -17,20 +17,20 @@ namespace vision { using namespace facebook; using namespace jni; -using TSelf = local_ref; +using TSelf = local_ref; -TSelf CameraView::initHybrid(alias_ref jThis) { +TSelf JCameraView::initHybrid(alias_ref jThis) { return makeCxxInstance(jThis); } -void CameraView::registerNatives() { +void JCameraView::registerNatives() { registerHybrid({ - makeNativeMethod("initHybrid", CameraView::initHybrid), - makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback), + makeNativeMethod("initHybrid", JCameraView::initHybrid), + makeNativeMethod("frameProcessorCallback", JCameraView::frameProcessorCallback), }); } -void CameraView::frameProcessorCallback(const alias_ref& frame) { +void JCameraView::frameProcessorCallback(const alias_ref& frame) { if (frameProcessor_ == nullptr) { __android_log_write(ANDROID_LOG_WARN, TAG, "Called Frame Processor callback, but `frameProcessor` is null!"); return; @@ -47,11 +47,11 @@ void CameraView::frameProcessorCallback(const alias_ref } } -void CameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) { +void JCameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) { frameProcessor_ = frameProcessor; } -void vision::CameraView::unsetFrameProcessor() { +void vision::JCameraView::unsetFrameProcessor() { frameProcessor_ = nullptr; } diff --git a/android/src/main/cpp/CameraView.h b/android/src/main/cpp/java-bindings/JCameraView.h similarity index 83% rename from android/src/main/cpp/CameraView.h rename to android/src/main/cpp/java-bindings/JCameraView.h index a1ca070d76..817f20828c 100644 --- a/android/src/main/cpp/CameraView.h +++ b/android/src/main/cpp/java-bindings/JCameraView.h @@ -16,7 +16,7 @@ namespace vision { using namespace facebook; using TFrameProcessor = std::function)>; -class CameraView : public jni::HybridClass { +class JCameraView : public jni::HybridClass { public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;"; static auto constexpr TAG = "VisionCamera"; @@ -29,12 +29,12 @@ class CameraView : public jni::HybridClass { private: friend HybridBase; - jni::global_ref javaPart_; + jni::global_ref javaPart_; TFrameProcessor frameProcessor_; void frameProcessorCallback(const jni::alias_ref& frame); - explicit CameraView(jni::alias_ref jThis) : + explicit JCameraView(jni::alias_ref jThis) : javaPart_(jni::make_global(jThis)), frameProcessor_(nullptr) {} diff --git a/android/src/main/cpp/java-bindings/JFrame.cpp b/android/src/main/cpp/java-bindings/JFrame.cpp new file mode 100644 index 0000000000..24752b0de3 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrame.cpp @@ -0,0 +1,65 @@ +// +// Created by Marc on 21.07.2023. +// + +#include "JFrame.h" + +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +int JFrame::getWidth() const { + static const auto getWidthMethod = getClass()->getMethod("getWidth"); + return getWidthMethod(self()); +} + +int JFrame::getHeight() const { + static const auto getWidthMethod = getClass()->getMethod("getHeight"); + return getWidthMethod(self()); +} + +bool JFrame::getIsValid() const { + static const auto getIsValidMethod = getClass()->getMethod("getIsValid"); + return getIsValidMethod(self()); +} + +bool JFrame::getIsMirrored() const { + static const auto getIsMirroredMethod = getClass()->getMethod("getIsMirrored"); + return getIsMirroredMethod(self()); +} + +jlong JFrame::getTimestamp() const { + static const auto getTimestampMethod = getClass()->getMethod("getTimestamp"); + return getTimestampMethod(self()); +} + +local_ref JFrame::getOrientation() const { + static const auto getOrientationMethod = getClass()->getMethod("getOrientation"); + return getOrientationMethod(self()); +} + +int JFrame::getPlanesCount() const { + static const auto getPlanesCountMethod = getClass()->getMethod("getPlanesCount"); + return getPlanesCountMethod(self()); +} + +int JFrame::getBytesPerRow() const { + static const auto getBytesPerRowMethod = getClass()->getMethod("getBytesPerRow"); + return getBytesPerRowMethod(self()); +} + +local_ref JFrame::toByteArray() const { + static const auto toByteArrayMethodMethod = getClass()->getMethod("toByteArray"); + return toByteArrayMethod(self()); +} + +void JFrame::close() { + static const auto closeMethod = getClass()->getMethod("close"); + closeMethod(self()); +} + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JImageProxy.h b/android/src/main/cpp/java-bindings/JFrame.h similarity index 73% rename from android/src/main/cpp/java-bindings/JImageProxy.h rename to android/src/main/cpp/java-bindings/JFrame.h index 81cb5f25eb..8d9949bcc0 100644 --- a/android/src/main/cpp/java-bindings/JImageProxy.h +++ b/android/src/main/cpp/java-bindings/JFrame.h @@ -1,5 +1,5 @@ // -// Created by Marc on 19/06/2021. +// Created by Marc on 21.07.2023. // #pragma once @@ -12,8 +12,8 @@ namespace vision { using namespace facebook; using namespace jni; -struct JImageProxy : public JavaClass { - static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;"; +struct JFrame : public JavaClass { + static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;"; public: int getWidth() const; diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp new file mode 100644 index 0000000000..23418852f3 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp @@ -0,0 +1,28 @@ +// +// Created by Marc Rousavy on 29.09.21. +// + +#include "JFrameProcessor.h" + +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +local_ref JFrameProcessorPlugin::callback(alias_ref image, + alias_ref> params) const { + auto callbackMethod = getClass()->getMethod("callback"); + + auto result = callbackMethod(self(), image, params); + return make_local(result); +} + +std::string JFrameProcessorPlugin::getName() const { + auto getNameMethod = getClass()->getMethod("getName"); + return getNameMethod(self())->toStdString(); +} + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.h b/android/src/main/cpp/java-bindings/JFrameProcessor.h new file mode 100644 index 0000000000..32b8d1c0f6 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.h @@ -0,0 +1,40 @@ +// +// Created by Marc Rousavy on 29.09.21 +// + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "JFrame.h" + +namespace vision { + +using namespace facebook; + +struct JFrameProcessor : public jni::HybridClass { +public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;"; + static jni::local_ref initHybrid(jni::alias_ref jThis); + static void registerNatives(); + explicit JFrameProcessor(std::shared_ptr worklet, + std::shared_ptr context); + +public: + /** + * Call the JS Frame Processor. + */ + void call(alias_ref frame) const; + +private: + friend HybridBase; + jni::global_ref javaPart_; +}; + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp index 54a162b9ac..92753b43eb 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp @@ -12,13 +12,13 @@ namespace vision { using namespace facebook; using namespace jni; -using TCallback = jobject(alias_ref, alias_ref>); +using TCallback = jobject(alias_ref, alias_ref); -local_ref JFrameProcessorPlugin::callback(alias_ref image, - alias_ref> params) const { +local_ref JFrameProcessorPlugin::callback(alias_ref frame, + alias_ref params) const { auto callbackMethod = getClass()->getMethod("callback"); - auto result = callbackMethod(self(), image, params); + auto result = callbackMethod(self(), frame, params); return make_local(result); } diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index ad66c5d54a..dc16396ef4 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -8,7 +8,7 @@ #include #include -#include "JImageProxy.h" +#include "JFrame.h" namespace vision { @@ -22,8 +22,8 @@ struct JFrameProcessorPlugin : public JavaClass { /** * Call the plugin. */ - local_ref callback(alias_ref image, - alias_ref> params) const; + local_ref callback(alias_ref frame, + alias_ref params) const; /** * Get the user-defined name of the Frame Processor Plugin */ diff --git a/android/src/main/cpp/java-bindings/JImageProxy.cpp b/android/src/main/cpp/java-bindings/JImageProxy.cpp deleted file mode 100644 index 4c7b210e48..0000000000 --- a/android/src/main/cpp/java-bindings/JImageProxy.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by Marc Rousavy on 22.06.21. -// - -#include "JImageProxy.h" - -#include -#include - -namespace vision { - -using namespace facebook; -using namespace jni; - -int JImageProxy::getWidth() const { - static const auto getWidthMethod = getClass()->getMethod("getWidth"); - return getWidthMethod(self()); -} - -int JImageProxy::getHeight() const { - static const auto getWidthMethod = getClass()->getMethod("getHeight"); - return getWidthMethod(self()); -} - -alias_ref getUtilsClass() { - static const auto ImageProxyUtilsClass = findClassStatic("com/mrousavy/camera/frameprocessor/ImageProxyUtils"); - return ImageProxyUtilsClass; -} - -bool JImageProxy::getIsValid() const { - auto utilsClass = getUtilsClass(); - static const auto isImageProxyValidMethod = utilsClass->getStaticMethod("isImageProxyValid"); - return isImageProxyValidMethod(utilsClass, self()); -} - -bool JImageProxy::getIsMirrored() const { - auto utilsClass = getUtilsClass(); - static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod("isImageProxyMirrored"); - return isImageProxyMirroredMethod(utilsClass, self()); -} - -jlong JImageProxy::getTimestamp() const { - auto utilsClass = getUtilsClass(); - static const auto getTimestampMethod = utilsClass->getStaticMethod("getTimestamp"); - return getTimestampMethod(utilsClass, self()); -} - -local_ref JImageProxy::getOrientation() const { - auto utilsClass = getUtilsClass(); - static const auto getOrientationMethod = utilsClass->getStaticMethod("getOrientation"); - return getOrientationMethod(utilsClass, self()); -} - -int JImageProxy::getPlanesCount() const { - auto utilsClass = getUtilsClass(); - static const auto getPlanesCountMethod = utilsClass->getStaticMethod("getPlanesCount"); - return getPlanesCountMethod(utilsClass, self()); -} - -int JImageProxy::getBytesPerRow() const { - auto utilsClass = getUtilsClass(); - static const auto getBytesPerRowMethod = utilsClass->getStaticMethod("getBytesPerRow"); - return getBytesPerRowMethod(utilsClass, self()); -} - -local_ref JImageProxy::toByteArray() const { - auto utilsClass = getUtilsClass(); - - static const auto toByteArrayMethod = utilsClass->getStaticMethod("toByteArray"); - return toByteArrayMethod(utilsClass, self()); -} - -void JImageProxy::close() { - static const auto closeMethod = getClass()->getMethod("close"); - closeMethod(self()); -} - -} // namespace vision diff --git a/android/src/main/cpp/VisionCameraScheduler.cpp b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp similarity index 52% rename from android/src/main/cpp/VisionCameraScheduler.cpp rename to android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp index 790ef07380..6e10502e66 100644 --- a/android/src/main/cpp/VisionCameraScheduler.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp @@ -2,31 +2,31 @@ // Created by Marc Rousavy on 25.07.21. // -#include "VisionCameraScheduler.h" +#include "JVisionCameraScheduler.h" #include namespace vision { using namespace facebook; -using TSelf = jni::local_ref; +using TSelf = jni::local_ref; -TSelf VisionCameraScheduler::initHybrid(jni::alias_ref jThis) { +TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref jThis) { return makeCxxInstance(jThis); } -void VisionCameraScheduler::dispatchAsync(std::function job) { +void JVisionCameraScheduler::dispatchAsync(std::function job) { // 1. add job to queue _jobs.push(job); scheduleTrigger(); } -void VisionCameraScheduler::scheduleTrigger() { +void JVisionCameraScheduler::scheduleTrigger() { // 2. schedule `triggerUI` to be called on the java thread static auto method = javaPart_->getClass()->getMethod("scheduleTrigger"); method(javaPart_.get()); } -void VisionCameraScheduler::trigger() { +void JVisionCameraScheduler::trigger() { std::unique_lock lock(_mutex); // 3. call job we enqueued in step 1. auto job = _jobs.front(); @@ -34,10 +34,10 @@ void VisionCameraScheduler::trigger() { _jobs.pop(); } -void VisionCameraScheduler::registerNatives() { +void JVisionCameraScheduler::registerNatives() { registerHybrid({ - makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid), - makeNativeMethod("trigger", VisionCameraScheduler::trigger), + makeNativeMethod("initHybrid", JVisionCameraScheduler::initHybrid), + makeNativeMethod("trigger", JVisionCameraScheduler::trigger), }); } diff --git a/android/src/main/cpp/VisionCameraScheduler.h b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h similarity index 85% rename from android/src/main/cpp/VisionCameraScheduler.h rename to android/src/main/cpp/java-bindings/JVisionCameraScheduler.h index 59f48e8038..ca4366e12b 100644 --- a/android/src/main/cpp/VisionCameraScheduler.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h @@ -23,7 +23,7 @@ using namespace facebook; * 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there * 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1. */ -class VisionCameraScheduler : public jni::HybridClass { +class JVisionCameraScheduler : public jni::HybridClass { public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;"; static jni::local_ref initHybrid(jni::alias_ref jThis); @@ -34,11 +34,11 @@ class VisionCameraScheduler : public jni::HybridClass { private: friend HybridBase; - jni::global_ref javaPart_; + jni::global_ref javaPart_; std::queue> _jobs; std::mutex _mutex; - explicit VisionCameraScheduler(jni::alias_ref jThis): + explicit JVisionCameraScheduler(jni::alias_ref jThis): javaPart_(jni::make_global(jThis)) {} // Schedules a call to `trigger` on the VisionCamera FP Thread diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index fcd1b82b82..df88eaafac 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -24,15 +24,12 @@ import androidx.lifecycle.* import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.* -import com.facebook.react.uimanager.events.RCTEventEmitter -import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlinx.coroutines.guava.await import java.lang.IllegalArgumentException import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import kotlin.math.floor import kotlin.math.max import kotlin.math.min diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index a229a1c62f..0c889f6e47 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -3,16 +3,11 @@ package com.mrousavy.camera import android.Manifest import android.content.Context import android.content.pm.PackageManager -import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.os.Build import android.util.Log -import android.util.Size -import androidx.camera.core.CameraSelector -import androidx.camera.extensions.ExtensionMode import androidx.camera.extensions.ExtensionsManager import androidx.camera.lifecycle.ProcessCameraProvider -import androidx.camera.video.QualitySelector import androidx.core.content.ContextCompat import com.facebook.react.bridge.* import com.facebook.react.module.annotations.ReactModule @@ -21,7 +16,7 @@ import com.facebook.react.modules.core.PermissionListener import com.facebook.react.uimanager.UIManagerHelper import com.facebook.react.bridge.ReactApplicationContext import java.util.concurrent.ExecutorService -import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager +import com.mrousavy.camera.frameprocessor.VisionCameraProxy import com.mrousavy.camera.parsers.* import com.mrousavy.camera.utils.* import kotlinx.coroutines.* @@ -30,40 +25,23 @@ import java.util.concurrent.Executors @ReactModule(name = CameraViewModule.TAG) @Suppress("unused") -class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) { +class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJavaModule(reactContext) { companion object { const val TAG = "CameraView" var RequestCode = 10 - - fun parsePermissionStatus(status: Int): String { - return when (status) { - PackageManager.PERMISSION_DENIED -> "denied" - PackageManager.PERMISSION_GRANTED -> "authorized" - else -> "not-determined" - } - } } var frameProcessorThread: ExecutorService = Executors.newSingleThreadExecutor() private val coroutineScope = CoroutineScope(Dispatchers.Default) // TODO: or Dispatchers.Main? - private var frameProcessorManager: FrameProcessorRuntimeManager? = null - private fun cleanup() { + override fun invalidate() { + super.invalidate() + frameProcessorThread.shutdown() if (coroutineScope.isActive) { coroutineScope.cancel("CameraViewModule has been destroyed.") } } - override fun onCatalystInstanceDestroy() { - super.onCatalystInstanceDestroy() - cleanup() - } - - override fun invalidate() { - super.invalidate() - cleanup() - } - override fun getName(): String { return TAG } @@ -75,6 +53,18 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase return view ?: throw ViewNotFoundError(viewId) } + @ReactMethod(isBlockingSynchronousMethod = true) + fun installFrameProcessorBindings(): Boolean { + try { + frameProcessorManager = VisionCameraProxy(reactApplicationContext, frameProcessorThread) + frameProcessorManager!!.installBindings() + return true + } catch (e: Error) { + Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e) + return false + } + } + @ReactMethod fun takePhoto(viewTag: Int, options: ReadableMap, promise: Promise) { coroutineScope.launch { @@ -151,18 +141,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase } } - @ReactMethod(isBlockingSynchronousMethod = true) - fun installFrameProcessorBindings(): Boolean { - try { - frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext, frameProcessorThread) - frameProcessorManager!!.installBindings() - return true - } catch (e: Error) { - Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e) - return false - } - } - @ReactMethod fun getAvailableCameraDevices(promise: Promise) { coroutineScope.launch { diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java new file mode 100644 index 0000000000..d5adf5c0b5 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -0,0 +1,130 @@ +package com.mrousavy.camera.frameprocessor; + +import android.annotation.SuppressLint; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.media.Image; + +import androidx.annotation.Keep; +import androidx.camera.core.ImageProxy; + +import com.facebook.jni.HybridData; +import com.facebook.proguard.annotations.DoNotStrip; + +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; + +@SuppressWarnings("JavaJniMissingFunction") // using fbjni here +public class Frame { + @SuppressWarnings({"unused", "FieldCanBeLocal"}) + @DoNotStrip + private final HybridData mHybridData; + private final ImageProxy imageProxy; + + public Frame(ImageProxy imageProxy) { + this.imageProxy = imageProxy; + mHybridData = initHybrid(); + } + + private native HybridData initHybrid(); + + + @SuppressWarnings("unused") + @DoNotStrip + private int getWidth() { + return imageProxy.getWidth(); + } + + @SuppressWarnings("unused") + @DoNotStrip + private int getHeight() { + return imageProxy.getHeight(); + } + + @SuppressWarnings("unused") + @DoNotStrip + private boolean getIsValid() { + try { + @SuppressLint("UnsafeOptInUsageError") + Image image = imageProxy.getImage(); + if (image == null) return false; + // will throw an exception if the image is already closed + image.getCropRect(); + // no exception thrown, image must still be valid. + return true; + } catch (Exception e) { + // exception thrown, image has already been closed. + return false; + } + } + + @SuppressWarnings("unused") + @DoNotStrip + private boolean getIsMirrored() { + Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); + // TODO: Figure out how to get isMirrored from ImageProxy + return false; + } + + @SuppressWarnings("unused") + @DoNotStrip + private long getTimestamp() { + return imageProxy.getImageInfo().getTimestamp(); + } + + @SuppressWarnings("unused") + @DoNotStrip + private String getOrientation() { + int rotation = imageProxy.getImageInfo().getRotationDegrees(); + if (rotation >= 45 && rotation < 135) + return "landscapeRight"; + if (rotation >= 135 && rotation < 225) + return "portraitUpsideDown"; + if (rotation >= 225 && rotation < 315) + return "landscapeLeft"; + return "portrait"; + } + + @SuppressWarnings("unused") + @DoNotStrip + private int getPlanesCount() { + return imageProxy.getPlanes().length; + } + + @SuppressWarnings("unused") + @DoNotStrip + private int getBytesPerRow() { + return imageProxy.getPlanes()[0].getRowStride(); + } + + private static byte[] byteArrayCache; + + @SuppressWarnings("unused") + @DoNotStrip + private byte[] toByteArray() { + switch (imageProxy.getFormat()) { + case ImageFormat.YUV_420_888: + ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); + ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer(); + int ySize = yBuffer.remaining(); + int vuSize = vuBuffer.remaining(); + + if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) { + byteArrayCache = new byte[ySize + vuSize]; + } + + yBuffer.get(byteArrayCache, 0, ySize); + vuBuffer.get(byteArrayCache, ySize, vuSize); + + return byteArrayCache; + default: + throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!"); + } + } + + @SuppressWarnings("unused") + @DoNotStrip + private void close() { + imageProxy.close(); + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java new file mode 100644 index 0000000000..32da301910 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java @@ -0,0 +1,16 @@ +package com.mrousavy.camera.frameprocessor; + +import androidx.annotation.Keep; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.facebook.proguard.annotations.DoNotStrip; + +/** + * Represents a JS Frame Processor + */ +public abstract class FrameProcessor { + /** + * Call the JS Frame Processor function with the given Frame + */ + public native void call(Frame frame); +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java index 98905d5ed7..eaaf636402 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java @@ -48,6 +48,6 @@ protected FrameProcessorPlugin(@NonNull String name) { * @param plugin An instance of a plugin. */ public static void register(@NonNull FrameProcessorPlugin plugin) { - FrameProcessorRuntimeManager.Companion.addPlugin(plugin); + VisionCameraProxy.Companion.addPlugin(plugin); } } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java deleted file mode 100644 index f1cf0a1f3e..0000000000 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.mrousavy.camera.frameprocessor; - -import android.annotation.SuppressLint; -import android.graphics.ImageFormat; -import android.graphics.Matrix; -import android.media.Image; - -import androidx.annotation.Keep; -import androidx.camera.core.ImageProxy; -import com.facebook.proguard.annotations.DoNotStrip; - -import java.nio.ByteBuffer; - -@SuppressWarnings("unused") // used through JNI -@DoNotStrip -@Keep -public class ImageProxyUtils { - @SuppressLint("UnsafeOptInUsageError") - @DoNotStrip - @Keep - public static boolean isImageProxyValid(ImageProxy imageProxy) { - try { - Image image = imageProxy.getImage(); - if (image == null) return false; - // will throw an exception if the image is already closed - image.getCropRect(); - // no exception thrown, image must still be valid. - return true; - } catch (Exception e) { - // exception thrown, image has already been closed. - return false; - } - } - - @DoNotStrip - @Keep - public static boolean isImageProxyMirrored(ImageProxy imageProxy) { - Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); - // TODO: Figure out how to get isMirrored from ImageProxy - return false; - } - - @DoNotStrip - @Keep - public static String getOrientation(ImageProxy imageProxy) { - int rotation = imageProxy.getImageInfo().getRotationDegrees(); - if (rotation >= 45 && rotation < 135) - return "landscapeRight"; - if (rotation >= 135 && rotation < 225) - return "portraitUpsideDown"; - if (rotation >= 225 && rotation < 315) - return "landscapeLeft"; - return "portrait"; - } - - @DoNotStrip - @Keep - public static long getTimestamp(ImageProxy imageProxy) { - return imageProxy.getImageInfo().getTimestamp(); - } - - @DoNotStrip - @Keep - public static int getPlanesCount(ImageProxy imageProxy) { - return imageProxy.getPlanes().length; - } - - @DoNotStrip - @Keep - public static int getBytesPerRow(ImageProxy imageProxy) { - return imageProxy.getPlanes()[0].getRowStride(); - } - - private static byte[] byteArrayCache; - - @DoNotStrip - @Keep - public static byte[] toByteArray(ImageProxy imageProxy) { - switch (imageProxy.getFormat()) { - case ImageFormat.YUV_420_888: - ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); - ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer(); - int ySize = yBuffer.remaining(); - int vuSize = vuBuffer.remaining(); - - if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) { - byteArrayCache = new byte[ySize + vuSize]; - } - - yBuffer.get(byteArrayCache, 0, ySize); - vuBuffer.get(byteArrayCache, ySize, vuSize); - - return byteArrayCache; - default: - throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!"); - } - } -} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java new file mode 100644 index 0000000000..f99e5a8929 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java @@ -0,0 +1,9 @@ +package com.mrousavy.camera.frameprocessor; + +import com.mrousavy.camera.CameraViewModule; + +public class VisionCameraInstaller { + private native void install() { + + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt similarity index 88% rename from android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt rename to android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index b6abf7eb99..e9f57a0c48 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -13,10 +13,9 @@ import java.lang.ref.WeakReference import java.util.concurrent.ExecutorService @Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that. -class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { +class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { companion object { - const val TAG = "FrameProcessorRuntime" - private val Plugins: ArrayList = ArrayList() + const val TAG = "VisionCameraProxy" init { try { @@ -26,10 +25,6 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProces throw e } } - - fun addPlugin(plugin: FrameProcessorPlugin) { - Plugins.add(plugin) - } } @DoNotStrip diff --git a/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt new file mode 100644 index 0000000000..a85a7bc605 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt @@ -0,0 +1,11 @@ +package com.mrousavy.camera.parsers + +import android.content.pm.PackageManager + +fun parsePermissionStatus(status: Int): String { + return when (status) { + PackageManager.PERMISSION_DENIED -> "denied" + PackageManager.PERMISSION_GRANTED -> "authorized" + else -> "not-determined" + } +} diff --git a/ios/Frame Processor/FrameProcessorPluginHostObject.mm b/ios/Frame Processor/FrameProcessorPluginHostObject.mm index a91b5f85c0..ce631ce2c3 100644 --- a/ios/Frame Processor/FrameProcessorPluginHostObject.mm +++ b/ios/Frame Processor/FrameProcessorPluginHostObject.mm @@ -12,6 +12,8 @@ #import "FrameHostObject.h" #import "JSINSObjectConversion.h" +using namespace facebook; + std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) { std::vector result; result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); diff --git a/ios/Frame Processor/VisionCameraProxy.mm b/ios/Frame Processor/VisionCameraProxy.mm index 784570726c..81cecfd87c 100644 --- a/ios/Frame Processor/VisionCameraProxy.mm +++ b/ios/Frame Processor/VisionCameraProxy.mm @@ -177,7 +177,7 @@ - (SkiaRenderer* _Nonnull)getSkiaRenderer; const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { - if (count != 1 || !arguments[0].isString()) { + if (count < 1 || !arguments[0].isString()) { throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); } auto pluginName = arguments[0].asString(runtime).utf8(runtime);