Skip to content

Commit 84ef60a

Browse files
davidgoveadavidgovea
committed
feat(mlkit): add on-device image labeling from ML Kit
1 parent 39fcc51 commit 84ef60a

16 files changed

Lines changed: 619 additions & 14 deletions

‎android/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,4 +77,5 @@ dependencies {
7777
mlkitImplementation "com.google.android.gms:play-services-mlkit-text-recognition:${safeExtGet('mlkit-text-recognition', '16.0.0')}"
7878
mlkitImplementation "com.google.mlkit:barcode-scanning:${safeExtGet('mlkit-barcode-scanning', '16.0.0')}"
7979
mlkitImplementation "com.google.mlkit:face-detection:${safeExtGet('mlkit-face-detection', '16.0.0')}"
80+
mlkitImplementation "com.google.mlkit:image-labeling:${safeExtGet('mlkit-image-labeling', '16.0.0')}"
8081
}

‎android/src/main/java/org/reactnative/camera/CameraViewManager.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ public enum Events {
2121
EVENT_ON_BAR_CODE_READ("onBarCodeRead"),
2222
EVENT_ON_FACES_DETECTED("onFacesDetected"),
2323
EVENT_ON_BARCODES_DETECTED("onGoogleVisionBarcodesDetected"),
24+
EVENT_ON_LABELS_DETECTED("onLabelsDetected"),
2425
EVENT_ON_FACE_DETECTION_ERROR("onFaceDetectionError"),
2526
EVENT_ON_BARCODE_DETECTION_ERROR("onGoogleVisionBarcodeDetectionError"),
27+
EVENT_ON_LABEL_DETECTION_ERROR("onLabelDetectionError"),
2628
EVENT_ON_TEXT_RECOGNIZED("onTextRecognized"),
2729
EVENT_ON_PICTURE_TAKEN("onPictureTaken"),
2830
EVENT_ON_PICTURE_SAVED("onPictureSaved"),
@@ -217,6 +219,11 @@ public void setTextRecognizing(RNCameraView view, boolean textRecognizerEnabled)
217219
view.setShouldRecognizeText(textRecognizerEnabled);
218220
}
219221

222+
@ReactProp(name = "labelDetectorEnabled")
223+
public void setLabelDetecting(RNCameraView view, boolean labelDetectorEnabled) {
224+
view.setShouldDetectLabels(labelDetectorEnabled);
225+
}
226+
220227
/**---limit scan area addition---**/
221228
@ReactProp(name = "rectOfInterest")
222229
public void setRectOfInterest(RNCameraView view, ReadableMap coordinates) {

‎android/src/main/java/org/reactnative/camera/RNCameraView.java‎

Lines changed: 53 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.reactnative.camera.tasks.*;
3131
import org.reactnative.camera.utils.RNFileUtils;
3232
import org.reactnative.facedetector.RNFaceDetector;
33+
import org.reactnative.imagelabeler.RNImageLabeler;
3334

3435
import java.io.ByteArrayOutputStream;
3536
import java.io.File;
@@ -39,7 +40,7 @@
3940
import java.util.concurrent.ConcurrentLinkedQueue;
4041

4142
public class RNCameraView extends CameraView implements LifecycleEventListener, BarCodeScannerAsyncTaskDelegate, FaceDetectorAsyncTaskDelegate,
42-
BarcodeDetectorAsyncTaskDelegate, TextRecognizerAsyncTaskDelegate, PictureSavedDelegate {
43+
BarcodeDetectorAsyncTaskDelegate, TextRecognizerAsyncTaskDelegate, ImageLabelerAsyncTaskDelegate, PictureSavedDelegate {
4344
private ThemedReactContext mThemedReactContext;
4445
private Queue<Promise> mPictureTakenPromises = new ConcurrentLinkedQueue<>();
4546
private Map<Promise, ReadableMap> mPictureTakenOptions = new ConcurrentHashMap<>();
@@ -64,16 +65,19 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
6465
public volatile boolean faceDetectorTaskLock = false;
6566
public volatile boolean googleBarcodeDetectorTaskLock = false;
6667
public volatile boolean textRecognizerTaskLock = false;
68+
public volatile boolean imageLabelerTaskLock = false;
6769

6870
// Scanning-related properties
6971
private MultiFormatReader mMultiFormatReader;
7072
private RNFaceDetector mFaceDetector;
7173
private RNBarcodeDetector mGoogleBarcodeDetector;
74+
private RNImageLabeler mImageLabeler;
7275
private boolean mShouldDetectFaces = false;
7376
private boolean mShouldGoogleDetectBarcodes = false;
7477
private boolean mShouldScanBarCodes = false;
7578
private boolean mShouldRecognizeText = false;
7679
private boolean mShouldDetectTouches = false;
80+
private boolean mShouldDetectLabels = false;
7781
private int mFaceDetectorMode = RNFaceDetector.FAST_MODE;
7882
private int mFaceDetectionLandmarks = RNFaceDetector.NO_LANDMARKS;
7983
private int mFaceDetectionClassifications = RNFaceDetector.NO_CLASSIFICATIONS;
@@ -166,7 +170,9 @@ public void onFramePreview(CameraView cameraView, byte[] data, int width, int he
166170
boolean willCallFaceTask = mShouldDetectFaces && !faceDetectorTaskLock && cameraView instanceof FaceDetectorAsyncTaskDelegate;
167171
boolean willCallGoogleBarcodeTask = mShouldGoogleDetectBarcodes && !googleBarcodeDetectorTaskLock && cameraView instanceof BarcodeDetectorAsyncTaskDelegate;
168172
boolean willCallTextTask = mShouldRecognizeText && !textRecognizerTaskLock && cameraView instanceof TextRecognizerAsyncTaskDelegate;
169-
if (!willCallBarCodeTask && !willCallFaceTask && !willCallGoogleBarcodeTask && !willCallTextTask) {
173+
boolean willCallLabelTask = mShouldDetectLabels && !imageLabelerTaskLock && cameraView instanceof ImageLabelerAsyncTaskDelegate;
174+
175+
if (!willCallBarCodeTask && !willCallFaceTask && !willCallGoogleBarcodeTask && !willCallTextTask && !willCallLabelTask) {
170176
return;
171177
}
172178

@@ -211,6 +217,12 @@ correctRotation, getResources().getDisplayMetrics().density, getFacing(),
211217
TextRecognizerAsyncTaskDelegate delegate = (TextRecognizerAsyncTaskDelegate) cameraView;
212218
new TextRecognizerAsyncTask(delegate, mThemedReactContext, data, width, height, correctRotation, getResources().getDisplayMetrics().density, getFacing(), getWidth(), getHeight(), mPaddingX, mPaddingY).execute();
213219
}
220+
221+
if (willCallLabelTask) {
222+
imageLabelerTaskLock = true;
223+
ImageLabelerAsyncTaskDelegate delegate = (ImageLabelerAsyncTaskDelegate) cameraView;
224+
new ImageLabelerAsyncTask(delegate, mImageLabeler, data, width, height, correctRotation, getResources().getDisplayMetrics().density, getFacing(), getWidth(), getHeight(), mPaddingX, mPaddingY).execute();
225+
}
214226
}
215227
});
216228
}
@@ -362,7 +374,7 @@ public void setShouldScanBarCodes(boolean shouldScanBarCodes) {
362374
initBarcodeReader();
363375
}
364376
this.mShouldScanBarCodes = shouldScanBarCodes;
365-
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
377+
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
366378
}
367379

368380
public void onBarCodeRead(Result barCode, int width, int height, byte[] imageData) {
@@ -483,7 +495,7 @@ public void setShouldDetectFaces(boolean shouldDetectFaces) {
483495
setupFaceDetector();
484496
}
485497
this.mShouldDetectFaces = shouldDetectFaces;
486-
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
498+
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
487499
}
488500

489501
public void onFacesDetected(WritableArray data) {
@@ -520,7 +532,7 @@ public void setShouldGoogleDetectBarcodes(boolean shouldDetectBarcodes) {
520532
setupBarcodeDetector();
521533
}
522534
this.mShouldGoogleDetectBarcodes = shouldDetectBarcodes;
523-
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
535+
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
524536
}
525537

526538
public void setGoogleVisionBarcodeType(int barcodeType) {
@@ -571,14 +583,49 @@ public void onBarcodeDetectingTaskCompleted() {
571583
googleBarcodeDetectorTaskLock = false;
572584
}
573585

586+
/**
587+
* Initial setup of the image labeler
588+
*/
589+
private void setupImageLabeler() {
590+
mImageLabeler = new RNImageLabeler(mThemedReactContext);
591+
}
592+
593+
public void setShouldDetectLabels(boolean shouldDetectLabels) {
594+
if (shouldDetectLabels && mImageLabeler == null) {
595+
setupImageLabeler();
596+
}
597+
this.mShouldDetectLabels = shouldDetectLabels;
598+
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
599+
}
600+
601+
public void onLabelsDetected(WritableArray labelsDetected) {
602+
if (!mShouldDetectLabels) {
603+
return;
604+
}
605+
RNCameraViewHelper.emitLabelsDetectedEvent(this, labelsDetected);
606+
}
607+
608+
public void onImageLabelingError(RNImageLabeler imageLabeler) {
609+
if (!mShouldDetectLabels) {
610+
return;
611+
}
612+
613+
RNCameraViewHelper.emitImageLabelingErrorEvent(this, imageLabeler);
614+
}
615+
616+
@Override
617+
public void onImageLabelingTaskCompleted() {
618+
imageLabelerTaskLock = false;
619+
}
620+
574621
/**
575622
*
576623
* Text recognition
577624
*/
578625

579626
public void setShouldRecognizeText(boolean shouldRecognizeText) {
580627
this.mShouldRecognizeText = shouldRecognizeText;
581-
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
628+
setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
582629
}
583630

584631
public void onTextRecognized(WritableArray serializedData) {

‎android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.reactnative.camera.events.*;
2020
import org.reactnative.barcodedetector.RNBarcodeDetector;
2121
import org.reactnative.facedetector.RNFaceDetector;
22+
import org.reactnative.imagelabeler.RNImageLabeler;
2223

2324
import java.text.SimpleDateFormat;
2425
import java.util.Calendar;
@@ -334,6 +335,32 @@ public void run() {
334335
});
335336
}
336337

338+
// Image labeling events
339+
340+
public static void emitLabelsDetectedEvent(final ViewGroup view, final WritableArray data) {
341+
342+
final ReactContext reactContext = (ReactContext) view.getContext();
343+
reactContext.runOnNativeModulesQueueThread(new Runnable() {
344+
@Override
345+
public void run() {
346+
LabelsDetectedEvent event = LabelsDetectedEvent.obtain(view.getId(), data);
347+
reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event);
348+
}
349+
});
350+
}
351+
352+
public static void emitImageLabelingErrorEvent(final ViewGroup view, final RNImageLabeler imageLabeler) {
353+
354+
final ReactContext reactContext = (ReactContext) view.getContext();
355+
reactContext.runOnNativeModulesQueueThread(new Runnable() {
356+
@Override
357+
public void run() {
358+
LabelDetectionErrorEvent event = LabelDetectionErrorEvent.obtain(view.getId(), imageLabeler);
359+
reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event);
360+
}
361+
});
362+
}
363+
337364
// Utilities
338365

339366
public static int getCorrectCameraRotation(int rotation, int facing, int cameraOrientation) {

‎android/src/main/java/org/reactnative/camera/events/LabelDetectionErrorEvent.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package org.reactnative.camera.events;
2+
3+
import androidx.core.util.Pools;
4+
5+
import com.facebook.react.bridge.Arguments;
6+
import com.facebook.react.bridge.WritableMap;
7+
import com.facebook.react.uimanager.events.Event;
8+
import com.facebook.react.uimanager.events.RCTEventEmitter;
9+
10+
import org.reactnative.camera.CameraViewManager;
11+
import org.reactnative.imagelabeler.RNImageLabeler;
12+
13+
public class LabelDetectionErrorEvent extends Event<LabelDetectionErrorEvent> {
14+
private static final Pools.SynchronizedPool<LabelDetectionErrorEvent> EVENTS_POOL = new Pools.SynchronizedPool<>(3);
15+
private RNImageLabeler mImageLabeler;
16+
17+
private LabelDetectionErrorEvent() {
18+
}
19+
20+
public static LabelDetectionErrorEvent obtain(int viewTag, RNImageLabeler imageLabeler) {
21+
LabelDetectionErrorEvent event = EVENTS_POOL.acquire();
22+
if (event == null) {
23+
event = new LabelDetectionErrorEvent();
24+
}
25+
event.init(viewTag, imageLabeler);
26+
return event;
27+
}
28+
29+
private void init(int viewTag, RNImageLabeler imageLabeler) {
30+
super.init(viewTag);
31+
mImageLabeler = imageLabeler;
32+
}
33+
34+
@Override
35+
public short getCoalescingKey() {
36+
return 0;
37+
}
38+
39+
@Override
40+
public String getEventName() {
41+
return CameraViewManager.Events.EVENT_ON_LABEL_DETECTION_ERROR.toString();
42+
}
43+
44+
@Override
45+
public void dispatch(RCTEventEmitter rctEventEmitter) {
46+
rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData());
47+
}
48+
49+
private WritableMap serializeEventData() {
50+
WritableMap map = Arguments.createMap();
51+
map.putBoolean("isOperational", mImageLabeler != null && mImageLabeler.isOperational());
52+
return map;
53+
}
54+
}

‎android/src/main/java/org/reactnative/camera/events/LabelsDetectedEvent.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package org.reactnative.camera.events;
2+
3+
import androidx.core.util.Pools;
4+
5+
import com.facebook.react.bridge.Arguments;
6+
import com.facebook.react.bridge.WritableArray;
7+
import com.facebook.react.bridge.WritableMap;
8+
import com.facebook.react.uimanager.events.Event;
9+
import com.facebook.react.uimanager.events.RCTEventEmitter;
10+
11+
import org.reactnative.camera.CameraViewManager;
12+
13+
public class LabelsDetectedEvent extends Event<LabelsDetectedEvent> {
14+
private static final Pools.SynchronizedPool<LabelsDetectedEvent> EVENTS_POOL =
15+
new Pools.SynchronizedPool<>(3);
16+
17+
private WritableArray mData;
18+
19+
private LabelsDetectedEvent() {}
20+
21+
public static LabelsDetectedEvent obtain(int viewTag, WritableArray data) {
22+
LabelsDetectedEvent event = EVENTS_POOL.acquire();
23+
if (event == null) {
24+
event = new LabelsDetectedEvent();
25+
}
26+
event.init(viewTag, data);
27+
return event;
28+
}
29+
30+
private void init(int viewTag, WritableArray data) {
31+
super.init(viewTag);
32+
mData = data;
33+
}
34+
35+
@Override
36+
public short getCoalescingKey() {
37+
if (mData.size() > Short.MAX_VALUE) {
38+
return Short.MAX_VALUE;
39+
}
40+
41+
return (short) mData.size();
42+
}
43+
44+
@Override
45+
public String getEventName() {
46+
return CameraViewManager.Events.EVENT_ON_LABELS_DETECTED.toString();
47+
}
48+
49+
@Override
50+
public void dispatch(RCTEventEmitter rctEventEmitter) {
51+
rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData());
52+
}
53+
54+
private WritableMap serializeEventData() {
55+
WritableMap event = Arguments.createMap();
56+
event.putString("type", "label");
57+
event.putArray("labels", mData);
58+
event.putInt("target", getViewTag());
59+
return event;
60+
}
61+
}

‎android/src/main/java/org/reactnative/camera/tasks/ImageLabelerAsyncTaskDelegate.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package org.reactnative.camera.tasks;
2+
3+
import com.facebook.react.bridge.WritableArray;
4+
import org.reactnative.imagelabeler.RNImageLabeler;
5+
6+
public interface ImageLabelerAsyncTaskDelegate {
7+
8+
void onLabelsDetected(WritableArray labels);
9+
10+
void onImageLabelingError(RNImageLabeler imageLabeler);
11+
12+
void onImageLabelingTaskCompleted();
13+
}

0 commit comments

Comments
 (0)