Skip to content

Commit aa31001

Browse files
feinsteinfeinstein
authored
feat: Add bitmap support to InputImage in commons package (#754)
1 parent f10bd5f commit aa31001

6 files changed

Lines changed: 213 additions & 5 deletions

File tree

‎packages/google_mlkit_commons/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 0.10.0
2+
3+
* Add support for bitmap data with `InputImage.fromBitmap()` constructor.
4+
15
## 0.9.0
26

37
* Update README.

‎packages/google_mlkit_commons/README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,19 @@ From bytes:
105105
final inputImage = InputImage.fromBytes(bytes: bytes, metadata: metadata);
106106
```
107107

108+
from bitmap data:
109+
110+
```dart
111+
final ui.Image image = await recorder.endRecording().toImage(width, height);
112+
final ByteData? byteData = await image.toByteData(format: ui.ImageByteFormat.rawRgba);
113+
final InputImage inputImage = InputImage.fromBitmap(
114+
bitmap: byteData!.buffer.asUint8List(),
115+
width: width,
116+
height: height,
117+
rotation: 0, // optional, defaults to 0, only used on Android
118+
);
119+
```
120+
108121
If you are using the [Camera plugin](https://pub.dev/packages/camera) make sure to configure your [CameraController](https://pub.dev/documentation/camera/latest/camera/CameraController-class.html) to only use `ImageFormatGroup.nv21` for Android and `ImageFormatGroup.bgra8888` for iOS.
109122

110123
Notice that the image rotation is computed in a different way for both iOS and Android. Image rotation is used in Android to convert the `InputImage` [from Dart to Java](https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java), but it is not used in iOS to convert [from Dart to Obj-C](https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/google_mlkit_commons/ios/Classes/MLKVisionImage%2BFlutterPlugin.m). However, image rotation and `camera.lensDirection` can be used in both platforms to [compensate x and y coordinates on a canvas](https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/example/lib/vision_detector_views/painters/coordinates_translator.dart).

‎packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,73 @@ public class InputImageConverter {
2020
public static InputImage getInputImageFromData(Map<String, Object> imageData,
2121
Context context,
2222
MethodChannel.Result result) {
23-
//Differentiates whether the image data is a path for a image file or contains image data in form of bytes
23+
//Differentiates whether the image data is a path for a image file, contains image data in form of bytes, or a bitmap
2424
String model = (String) imageData.get("type");
2525
InputImage inputImage;
26-
if (model != null && model.equals("file")) {
26+
if (model != null && model.equals("bitmap")) {
27+
try {
28+
byte[] bitmapData = (byte[]) imageData.get("bitmapData");
29+
if (bitmapData == null) {
30+
result.error("InputImageConverterError", "Bitmap data is null", null);
31+
return null;
32+
}
33+
34+
// Extract the rotation
35+
int rotation = 0;
36+
Object rotationObj = imageData.get("rotation");
37+
if (rotationObj != null) {
38+
rotation = (int) rotationObj;
39+
}
40+
41+
try {
42+
// Get metadata from the InputImage object if available
43+
Map<String, Object> metadataMap = (Map<String, Object>) imageData.get("metadata");
44+
if (metadataMap != null) {
45+
int width = Double.valueOf(Objects.requireNonNull(metadataMap.get("width")).toString()).intValue();
46+
int height = Double.valueOf(Objects.requireNonNull(metadataMap.get("height")).toString()).intValue();
47+
48+
// Create bitmap from the Flutter UI raw RGBA bytes
49+
android.graphics.Bitmap bitmap = android.graphics.Bitmap.createBitmap(width, height, android.graphics.Bitmap.Config.ARGB_8888);
50+
java.nio.IntBuffer intBuffer = java.nio.IntBuffer.allocate(bitmapData.length / 4);
51+
52+
// Convert RGBA bytes to int pixels
53+
for (int i = 0; i < bitmapData.length; i += 4) {
54+
int r = bitmapData[i] & 0xFF;
55+
int g = bitmapData[i + 1] & 0xFF;
56+
int b = bitmapData[i + 2] & 0xFF;
57+
int a = bitmapData[i + 3] & 0xFF;
58+
intBuffer.put((a << 24) | (r << 16) | (g << 8) | b);
59+
}
60+
intBuffer.rewind();
61+
62+
// Copy pixel data to bitmap
63+
bitmap.copyPixelsFromBuffer(intBuffer);
64+
return InputImage.fromBitmap(bitmap, rotation);
65+
}
66+
} catch (Exception e) {
67+
Log.e("ImageError", "Error creating bitmap from raw data", e);
68+
}
69+
70+
// Fallback: Try to decode as standard image format (JPEG, PNG)
71+
try {
72+
android.graphics.Bitmap bitmap = android.graphics.BitmapFactory.decodeByteArray(bitmapData, 0, bitmapData.length);
73+
if (bitmap == null) {
74+
result.error("InputImageConverterError", "Failed to decode bitmap from the provided data", null);
75+
return null;
76+
}
77+
return InputImage.fromBitmap(bitmap, rotation);
78+
} catch (Exception e) {
79+
Log.e("ImageError", "Getting Bitmap failed", e);
80+
result.error("InputImageConverterError", e.toString(), e);
81+
return null;
82+
}
83+
} catch (Exception e) {
84+
Log.e("ImageError", "Getting Bitmap failed");
85+
Log.e("ImageError", e.toString());
86+
result.error("InputImageConverterError", e.toString(), e);
87+
return null;
88+
}
89+
} else if (model != null && model.equals("file")) {
2790
try {
2891
inputImage = InputImage.fromFilePath(context, Uri.fromFile(new File(((String) imageData.get("path")))));
2992
return inputImage;

‎packages/google_mlkit_commons/ios/Classes/MLKVisionImage+FlutterPlugin.m‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ + (MLKVisionImage *)visionImageFromData:(NSDictionary *)imageData {
99
return [self filePathToVisionImage:imageData[@"path"]];
1010
} else if ([@"bytes" isEqualToString:imageType]) {
1111
return [self bytesToVisionImage:imageData];
12+
} else if ([@"bitmap" isEqualToString:imageType]) {
13+
return [self bitmapToVisionImage:imageData];
1214
} else {
1315
NSString *errorReason = [NSString stringWithFormat:@"No image type for: %@", imageType];
1416
@throw [NSException exceptionWithName:NSInvalidArgumentException
@@ -66,4 +68,67 @@ + (MLKVisionImage *)pixelBufferToVisionImage:(CVPixelBufferRef)pixelBufferRef {
6668
return [[MLKVisionImage alloc] initWithImage:uiImage];
6769
}
6870

71+
+ (MLKVisionImage *)bitmapToVisionImage:(NSDictionary *)imageDict {
72+
// Get the bitmap data
73+
FlutterStandardTypedData *bitmapData = imageDict[@"bitmapData"];
74+
75+
if (bitmapData == nil) {
76+
NSString *errorReason = @"Bitmap data is nil";
77+
@throw [NSException exceptionWithName:NSInvalidArgumentException
78+
reason:errorReason
79+
userInfo:nil];
80+
}
81+
82+
// Try to get metadata if available
83+
NSDictionary *metadata = imageDict[@"metadata"];
84+
if (metadata != nil) {
85+
NSNumber *width = metadata[@"width"];
86+
NSNumber *height = metadata[@"height"];
87+
88+
if (width != nil && height != nil) {
89+
// Create bitmap context from raw RGBA data
90+
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
91+
uint8_t *rawData = (uint8_t*)[bitmapData.data bytes];
92+
size_t bytesPerPixel = 4;
93+
size_t bytesPerRow = bytesPerPixel * width.intValue;
94+
size_t bitsPerComponent = 8;
95+
96+
CGContextRef context = CGBitmapContextCreate(rawData, width.intValue, height.intValue,
97+
bitsPerComponent, bytesPerRow, colorSpace,
98+
kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big);
99+
100+
if (context) {
101+
CGImageRef imageRef = CGBitmapContextCreateImage(context);
102+
UIImage *image = [UIImage imageWithCGImage:imageRef];
103+
104+
CGImageRelease(imageRef);
105+
CGContextRelease(context);
106+
CGColorSpaceRelease(colorSpace);
107+
108+
if (image) {
109+
MLKVisionImage *visionImage = [[MLKVisionImage alloc] initWithImage:image];
110+
visionImage.orientation = image.imageOrientation;
111+
return visionImage;
112+
}
113+
}
114+
115+
CGColorSpaceRelease(colorSpace);
116+
}
117+
}
118+
119+
// Fallback: try to create UIImage directly from data
120+
UIImage *image = [UIImage imageWithData:bitmapData.data];
121+
122+
if (image == nil) {
123+
NSString *errorReason = @"Failed to create UIImage from bitmap data";
124+
@throw [NSException exceptionWithName:NSInvalidArgumentException
125+
reason:errorReason
126+
userInfo:nil];
127+
}
128+
129+
MLKVisionImage *visionImage = [[MLKVisionImage alloc] initWithImage:image];
130+
visionImage.orientation = image.imageOrientation;
131+
return visionImage;
132+
}
133+
69134
@end

‎packages/google_mlkit_commons/lib/src/input_image.dart‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,26 @@ class InputImage {
1010
/// The bytes of the image.
1111
final Uint8List? bytes;
1212

13+
/// Raw bitmap pixel data.
14+
final Uint8List? bitmapData;
15+
1316
/// The type of image.
1417
final InputImageType type;
1518

1619
/// The image data when creating an image of type = [InputImageType.bytes].
1720
final InputImageMetadata? metadata;
1821

19-
InputImage._({this.filePath, this.bytes, required this.type, this.metadata});
22+
/// The rotation degrees for bitmap images.
23+
final int? rotation;
24+
25+
InputImage._({
26+
this.filePath,
27+
this.bytes,
28+
this.bitmapData,
29+
required this.type,
30+
this.metadata,
31+
this.rotation,
32+
});
2033

2134
/// Creates an instance of [InputImage] from path of image stored in device.
2235
factory InputImage.fromFilePath(String path) {
@@ -35,19 +48,69 @@ class InputImage {
3548
bytes: bytes, type: InputImageType.bytes, metadata: metadata);
3649
}
3750

51+
/// Creates an instance of [InputImage] from bitmap data.
52+
///
53+
/// This constructor is designed to work with bitmap data from Flutter UI components
54+
/// such as those obtained from ui.Image.toByteData(format: ui.ImageByteFormat.rawRgba).
55+
///
56+
/// Example usage with a RepaintBoundary:
57+
/// ```dart
58+
/// // Get the RenderObject from a GlobalKey
59+
/// final boundary = myKey.currentContext?.findRenderObject() as RenderRepaintBoundary?;
60+
/// // Capture the widget as an image
61+
/// final image = await boundary.toImage();
62+
/// // Get the raw RGBA bytes
63+
/// final byteData = await image.toByteData(format: ui.ImageByteFormat.rawRgba);
64+
/// // Create the InputImage
65+
/// final inputImage = InputImage.fromBitmap(
66+
/// bitmap: byteData!.buffer.asUint8List(),
67+
/// width: image.width,
68+
/// height: image.height,
69+
/// );
70+
/// ```
71+
///
72+
/// [bitmap] should be the raw bitmap data, typically from ui.Image.toByteData().
73+
/// [width] and [height] are the dimensions of the bitmap.
74+
/// [rotation] is optional and defaults to 0. It is only used on Android.
75+
factory InputImage.fromBitmap({
76+
required Uint8List bitmap,
77+
required int width,
78+
required int height,
79+
int rotation = 0,
80+
}) {
81+
return InputImage._(
82+
bitmapData: bitmap,
83+
type: InputImageType.bitmap,
84+
rotation: rotation,
85+
metadata: InputImageMetadata(
86+
size: Size(width.toDouble(), height.toDouble()),
87+
rotation: InputImageRotation.values.firstWhere(
88+
(element) => element.rawValue == rotation,
89+
orElse: () => InputImageRotation.rotation0deg,
90+
),
91+
// Assuming BGRA format from Flutter UI
92+
format: InputImageFormat.bgra8888,
93+
bytesPerRow: width * 4, // 4 bytes per pixel (RGBA)
94+
),
95+
);
96+
}
97+
3898
/// Returns a json representation of an instance of [InputImage].
3999
Map<String, dynamic> toJson() => {
40100
'bytes': bytes,
41101
'type': type.name,
42102
'path': filePath,
43-
'metadata': metadata?.toJson()
103+
'metadata': metadata?.toJson(),
104+
'bitmapData': bitmapData,
105+
'rotation': rotation
44106
};
45107
}
46108

47109
/// The type of [InputImage].
48110
enum InputImageType {
49111
file,
50112
bytes,
113+
bitmap,
51114
}
52115

53116
/// Data of image required when creating image from bytes.

‎packages/google_mlkit_commons/pubspec.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: google_mlkit_commons
22
description: A Flutter plugin with commons files to implement google's standalone ml kit made for mobile platform.
3-
version: 0.9.0
3+
version: 0.10.0
44
homepage: https://github.com/flutter-ml/google_ml_kit_flutter
55
repository: https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_commons
66

0 commit comments

Comments
 (0)