From a6a2d35a813213d5595466be91f1b4fb5d190eca Mon Sep 17 00:00:00 2001 From: Aaron Grider Date: Mon, 21 Feb 2022 14:27:04 -0800 Subject: [PATCH] Implement Android --- android/build.gradle | 3 +- .../OCRFrameProcessorPlugin.kt | 113 +++++++++++++++++- .../android/app/src/main/AndroidManifest.xml | 4 + ios/VisionCameraOcr.swift | 20 ---- 4 files changed, 113 insertions(+), 27 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index 63a9c82..6d81d85 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -29,7 +29,7 @@ android { compileSdkVersion getExtOrIntegerDefault('compileSdkVersion') buildToolsVersion getExtOrDefault('buildToolsVersion') defaultConfig { - minSdkVersion 16 + minSdkVersion 21 targetSdkVersion getExtOrIntegerDefault('targetSdkVersion') versionCode 1 versionName "1.0" @@ -129,5 +129,6 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" implementation project(':react-native-vision-camera') + implementation 'com.google.android.gms:play-services-mlkit-text-recognition:18.0.0' implementation "androidx.camera:camera-core:1.1.0-alpha08" } diff --git a/android/src/main/java/com/visioncameraocr/OCRFrameProcessorPlugin.kt b/android/src/main/java/com/visioncameraocr/OCRFrameProcessorPlugin.kt index 4c251fd..8ae6279 100644 --- a/android/src/main/java/com/visioncameraocr/OCRFrameProcessorPlugin.kt +++ b/android/src/main/java/com/visioncameraocr/OCRFrameProcessorPlugin.kt @@ -1,23 +1,124 @@ package com.visioncameraocr +import android.annotation.SuppressLint +import android.graphics.Point +import android.graphics.Rect +import android.media.Image import androidx.camera.core.ImageProxy import com.facebook.react.bridge.WritableNativeArray import com.facebook.react.bridge.WritableNativeMap +import com.google.android.gms.tasks.Task +import com.google.android.gms.tasks.Tasks +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.text.Text +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.text.latin.TextRecognizerOptions import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin - class OCRFrameProcessorPlugin: FrameProcessorPlugin("scanOCR") { - override fun callback(image: ImageProxy, params: Array): Any? { + private fun getBlockArray(blocks: MutableList): WritableNativeArray { + val blockArray = WritableNativeArray() + + for (block in blocks) { + val blockMap = WritableNativeMap() + + blockMap.putString("text", block.text) + blockMap.putArray("recognizedLanguages", getRecognizedLanguages(block.recognizedLanguage)) + blockMap.putArray("cornerPoints", block.cornerPoints?.let { getCornerPoints(it) }) + blockMap.putMap("frame", getFrame(block.boundingBox)) + blockMap.putArray("lines", getLineArray(block.lines)) + + blockArray.pushMap(blockMap) + } + return blockArray + } + + private fun getLineArray(lines: MutableList): WritableNativeArray { + val lineArray = WritableNativeArray() + + for (line in lines) { + val lineMap = WritableNativeMap() + + lineMap.putString("text", line.text) + lineMap.putArray("recognizedLanguages", getRecognizedLanguages(line.recognizedLanguage)) + lineMap.putArray("cornerPoints", line.cornerPoints?.let { getCornerPoints(it) }) + lineMap.putMap("frame", getFrame(line.boundingBox)) + lineMap.putArray("elements", getElementArray(line.elements)) + + lineArray.pushMap(lineMap) + } + return lineArray + } + + private fun getElementArray(elements: MutableList): WritableNativeArray { + val elementArray = WritableNativeArray() + + for (element in elements) { + val elementMap = WritableNativeMap() + + elementMap.putString("text", element.text) + elementMap.putArray("cornerPoints", element.cornerPoints?.let { getCornerPoints(it) }) + elementMap.putMap("frame", getFrame(element.boundingBox)) + } + return elementArray + } + + private fun getRecognizedLanguages(recognizedLanguage: String): WritableNativeArray { + val recognizedLanguages = WritableNativeArray() + recognizedLanguages.pushString(recognizedLanguage) + return recognizedLanguages + } + + private fun getCornerPoints(points: Array): WritableNativeArray { + val cornerPoints = WritableNativeArray() + + for (point in points) { + val pointMap = WritableNativeMap() + pointMap.putInt("x", point.x) + pointMap.putInt("y", point.y) + cornerPoints.pushMap(pointMap) + } + return cornerPoints + } + + private fun getFrame(boundingBox: Rect?): WritableNativeMap { + val frame = WritableNativeMap() + + if (boundingBox != null) { + frame.putDouble("x", boundingBox.exactCenterX().toDouble()) + frame.putDouble("y", boundingBox.exactCenterY().toDouble()) + frame.putInt("width", boundingBox.width()) + frame.putInt("height", boundingBox.height()) + frame.putInt("boundingCenterX", boundingBox.centerX()) + frame.putInt("boundingCenterY", boundingBox.centerY()) + } + return frame + } + + override fun callback(frame: ImageProxy, params: Array): Any? { + val result = WritableNativeMap() - result.putString("text", "Test Text") - val blocks = WritableNativeArray() - result.putArray("blocks", blocks) + val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) + + @SuppressLint("UnsafeOptInUsageError") + val mediaImage: Image? = frame.getImage() + + if (mediaImage != null) { + val image = InputImage.fromMediaImage(mediaImage, frame.imageInfo.rotationDegrees) + val task: Task = recognizer.process(image) + try { + val text: Text = Tasks.await(task) + result.putString("text", text.text) + result.putArray("blocks", getBlockArray(text.textBlocks)) + } catch (e: Exception) { + return null + } + } val data = WritableNativeMap() data.putMap("result", result) - return data } } \ No newline at end of file diff --git a/example/android/app/src/main/AndroidManifest.xml b/example/android/app/src/main/AndroidManifest.xml index f7626c2..be304e4 100644 --- a/example/android/app/src/main/AndroidManifest.xml +++ b/example/android/app/src/main/AndroidManifest.xml @@ -9,6 +9,10 @@ + + [String: CGFloat] { - - let offsetX = (frameRect.midX - ceil(frameRect.width)) / 2.0 - let offsetY = (frameRect.midY - ceil(frameRect.height)) / 2.0 - - let x = frameRect.maxX + offsetX - let y = frameRect.minY + offsetY - - return [ - "x": frameRect.origin.x, - "y": frameRect.origin.y, - "width": frameRect.width, - "height": frameRect.height, - "boundingCenterX": frameRect.midX, - "boundingCenterY": frameRect.midY - ] - } - */ - private static func getFrame(_ frameRect: CGRect) -> [String: CGFloat] { let offsetX = (frameRect.midX - ceil(frameRect.width)) / 2.0