Skip to content

Commit 4381620

Browse files
committed
fix(upstream): integrate open PRs and issues from flutter-ml/google_ml_kit_flutter
- PR flutter-ml#864 / Issue flutter-ml#863: Fix iOS IOSurface memory leak in pixelBufferToVisionImage. Replaces CoreImage CIContext+createCGImage round-trip with VisionImage(buffer:) via CMSampleBuffer wrapper, eliminating ~3.5 MiB leak per frame on sustained camera streaming. - PR flutter-ml#862 / Issue flutter-ml#825: Add Apple Silicon iOS 26+ simulator support. Ships opt-in Podfile helper (apple_silicon_simulator.rb + patch_arm64_simulator.py) that re-labels arm64 device slices as iOS Simulator and strips EXCLUDED_ARCHS. Updates README with usage instructions and wires up example app Podfile. - Issue flutter-ml#857: Fix IllegalStateException 'Reply already submitted' crash in DocumentScanner on Oppo/ColorOS devices. Clears pendingResult after calling .success() or .error() so duplicate onActivityResult deliveries are safe.
1 parent 58500ad commit 4381620

6 files changed

Lines changed: 270 additions & 11 deletions

File tree

packages/example/ios/Podfile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ def flutter_root
2323
end
2424

2525
require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root)
26+
require File.expand_path('../../google_mlkit_commons/ios/scripts/apple_silicon_simulator', __dir__)
2627

2728
flutter_ios_podfile_setup
2829

@@ -56,4 +57,6 @@ post_install do |installer|
5657
end
5758
end
5859
end
60+
61+
mlkit_apple_silicon_simulator_patch(installer)
5962
end

packages/google_mlkit_commons/README.md

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,33 @@ end
7777

7878
Notice that the minimum `IPHONEOS_DEPLOYMENT_TARGET` is 15.5, you can set it to something newer but not older.
7979

80+
#### Apple Silicon iOS Simulator (iOS 26+)
81+
82+
Google's `GoogleMLKit/*` pods only ship `arm64-iphoneos` and `x86_64-iphonesimulator` slices and exclude `arm64` from simulator builds. On Apple Silicon Macs running iOS 26+ simulators (where Rosetta 2 is no longer the default for the iOS Simulator) this makes `flutter run` fail with `Unable to find a destination matching the provided destination specifier`. Issue tracked upstream by Google: https://issuetracker.google.com/issues/178965151.
83+
84+
Until proper `arm64-iphonesimulator` slices are published, this plugin ships an **opt-in** Podfile helper that re-labels the existing arm64 device slice as iOS Simulator (the same `LC_BUILD_VERSION.platform` swap used by the well-known [`arm64-to-sim`](https://github.com/bogo/arm64-to-sim) tool) and strips the `EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64` from the generated xcconfigs.
85+
86+
To enable it, add two lines to your iOS `Podfile`:
87+
88+
```ruby
89+
# Near the top, after `require ... podhelper ...`:
90+
require File.expand_path(
91+
'.symlinks/plugins/google_mlkit_commons/ios/scripts/apple_silicon_simulator',
92+
__dir__,
93+
)
94+
95+
post_install do |installer|
96+
# ...your existing post_install code...
97+
98+
# Add this line at the end:
99+
mlkit_apple_silicon_simulator_patch(installer)
100+
end
101+
```
102+
103+
Then re-run `pod install`. The example app under `packages/example` is wired up this way.
104+
105+
The helper only changes vendored binaries inside `Pods/` and the `EXCLUDED_ARCHS` line in pod-generated xcconfigs. Device builds and release builds are unaffected. Remove the two lines to revert.
106+
80107
### Android
81108

82109
- minSdkVersion: 21

packages/google_mlkit_commons/ios/Classes/MLKVisionImage+FlutterPlugin.swift

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import Flutter
22
import MLKitVision
33
import UIKit
44
import CoreGraphics
5+
import CoreMedia
56
import CoreVideo
67

78
// MARK: - VisionImage from Flutter imageData
@@ -111,20 +112,46 @@ extension VisionImage {
111112
return pxBuffer
112113
}
113114

115+
// Wraps a CVPixelBuffer in a CMSampleBuffer and feeds it to MLKit via
116+
// VisionImage(buffer:). The previous implementation created a fresh CIContext
117+
// per frame and called createCGImage(_:from:); each call inserts a
118+
// CI::SurfaceCacheEntry that the system never releases under sustained
119+
// camera streaming, leaking ~3.5 MiB of IOSurface memory per call (300+ MiB
120+
// per minute on 720p BGRA). VisionImage(buffer:) bypasses CoreImage entirely
121+
// and is the same path Google's official MLKit iOS sample uses
122+
// (googlesamples/mlkit CameraViewController.swift).
114123
private static func pixelBufferToVisionImage(_ pixelBufferRef: CVPixelBuffer) -> VisionImage? {
115-
let ciImage = CIImage(cvPixelBuffer: pixelBufferRef)
116-
let context = CIContext(options: nil)
117-
let width = CVPixelBufferGetWidth(pixelBufferRef)
118-
let height = CVPixelBufferGetHeight(pixelBufferRef)
119-
guard let cgImage = context.createCGImage(
120-
ciImage,
121-
from: CGRect(x: 0, y: 0, width: width, height: height)
122-
) else {
124+
var formatDesc: CMVideoFormatDescription?
125+
let formatStatus = CMVideoFormatDescriptionCreateForImageBuffer(
126+
allocator: kCFAllocatorDefault,
127+
imageBuffer: pixelBufferRef,
128+
formatDescriptionOut: &formatDesc
129+
)
130+
guard formatStatus == noErr, let formatDescription = formatDesc else {
131+
return nil
132+
}
133+
134+
var timing = CMSampleTimingInfo(
135+
duration: .invalid,
136+
presentationTimeStamp: .zero,
137+
decodeTimeStamp: .invalid
138+
)
139+
var sampleBuffer: CMSampleBuffer?
140+
let sampleStatus = CMSampleBufferCreateForImageBuffer(
141+
allocator: kCFAllocatorDefault,
142+
imageBuffer: pixelBufferRef,
143+
dataReady: true,
144+
makeDataReadyCallback: nil,
145+
refcon: nil,
146+
formatDescription: formatDescription,
147+
sampleTiming: &timing,
148+
sampleBufferOut: &sampleBuffer
149+
)
150+
guard sampleStatus == noErr, let sampleBuffer = sampleBuffer else {
123151
return nil
124152
}
125-
// Swift ARC manages CGImage; UIImage(cgImage:) retains it.
126-
let uiImage = UIImage(cgImage: cgImage)
127-
return VisionImage(image: uiImage)
153+
154+
return VisionImage(buffer: sampleBuffer)
128155
}
129156

130157
private static func bitmapToVisionImage(_ imageDict: [String: Any]) -> VisionImage? {
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Opt-in Podfile helper that lets Google ML Kit pods build for Apple Silicon
2+
# iOS 26+ simulators. See packages/google_mlkit_commons/README.md (iOS
3+
# section) for rationale and usage. Upstream Google bug:
4+
# https://issuetracker.google.com/issues/178965151
5+
6+
def mlkit_apple_silicon_simulator_patch(installer)
7+
pods_dir = File.expand_path(installer.sandbox.root.to_s)
8+
patcher = File.expand_path('patch_arm64_simulator.py', __dir__)
9+
10+
framework_dirs = Dir.glob(File.join(pods_dir, '{MLKit*,MLImage*}'))
11+
.select { |d| File.directory?(d) }
12+
unless framework_dirs.empty?
13+
Pod::UI.puts ''
14+
Pod::UI.puts "[ml_kit] Patching #{framework_dirs.size} ML Kit " \
15+
'framework(s) for Apple Silicon iOS Simulator...'
16+
unless system('python3', patcher, *framework_dirs)
17+
Pod::UI.warn '[ml_kit] arm64 simulator patcher failed; ' \
18+
'simulator build may still require Rosetta.'
19+
end
20+
end
21+
22+
excluded = 'EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64'
23+
Dir.glob(File.join(pods_dir, 'Target Support Files', '**', '*.xcconfig'))
24+
.each do |xcconfig|
25+
text = File.read(xcconfig)
26+
new_text = text.lines.reject { |l| l.strip == excluded }.join
27+
File.write(xcconfig, new_text) if text != new_text
28+
end
29+
end
Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#!/usr/bin/env python3
2+
"""Re-label the arm64 device slice of Google ML Kit static frameworks as
3+
iOS Simulator. Walks every .o member of the arm64 archive and flips
4+
LC_BUILD_VERSION.platform from 2 (iOS) to 7 (iOS Simulator); no
5+
instructions or symbols are touched. Same approach as arm64-to-sim.
6+
Idempotent. See packages/google_mlkit_commons/README.md (iOS section).
7+
8+
Usage: python3 patch_arm64_simulator.py <Pods/MLKitFoo> [<Pods/MLKitBar> ...]
9+
"""
10+
11+
import os
12+
import struct
13+
import subprocess
14+
import sys
15+
import tempfile
16+
17+
FAT_MAGIC = 0xCAFEBABE
18+
FAT_MAGIC_64 = 0xCAFEBABF
19+
MH_MAGIC_64 = 0xFEEDFACF
20+
LC_BUILD_VERSION = 0x32
21+
PLATFORM_IOS = 2
22+
PLATFORM_IOS_SIMULATOR = 7
23+
CPU_TYPE_ARM64 = 0x0100000c
24+
25+
26+
def _patch_macho_object(buf):
27+
if len(buf) < 32:
28+
return buf, False
29+
magic = struct.unpack_from('<I', buf, 0)[0]
30+
if magic != MH_MAGIC_64:
31+
return buf, False
32+
cputype, _cpusub, _filetype, ncmds, _sizeofcmds, _flags, _reserved = \
33+
struct.unpack_from('<iIIIIII', buf, 4)
34+
if cputype != CPU_TYPE_ARM64:
35+
return buf, False
36+
new_buf = bytearray(buf)
37+
offset = 32
38+
patched = False
39+
for _ in range(ncmds):
40+
cmd, cmdsize = struct.unpack_from('<II', new_buf, offset)
41+
if cmd == LC_BUILD_VERSION:
42+
platform = struct.unpack_from('<I', new_buf, offset + 8)[0]
43+
if platform == PLATFORM_IOS:
44+
struct.pack_into('<I', new_buf, offset + 8,
45+
PLATFORM_IOS_SIMULATOR)
46+
patched = True
47+
offset += cmdsize
48+
return bytes(new_buf), patched
49+
50+
51+
def _patch_static_archive(archive_path):
52+
with open(archive_path, 'rb') as f:
53+
data = f.read()
54+
if data[:8] != b'!<arch>\n':
55+
return 0
56+
out = bytearray(data[:8])
57+
pos = 8
58+
n_patched = 0
59+
while pos + 60 <= len(data):
60+
header = data[pos:pos + 60]
61+
name = header[:16].rstrip().decode('ascii', errors='replace')
62+
try:
63+
size = int(header[48:58].rstrip().decode('ascii', errors='replace'))
64+
except ValueError:
65+
break
66+
body_start = pos + 60
67+
body_end = body_start + size
68+
body = data[body_start:body_end]
69+
if name.startswith('#1/'): # BSD long-name extension
70+
try:
71+
name_len = int(name[3:])
72+
except ValueError:
73+
name_len = 0
74+
obj_buf = data[body_start + name_len:body_end]
75+
new_obj, patched = _patch_macho_object(obj_buf)
76+
new_body = body[:name_len] + new_obj
77+
else:
78+
new_obj, patched = _patch_macho_object(body)
79+
new_body = new_obj
80+
if patched:
81+
n_patched += 1
82+
out += header + new_body
83+
pos = body_end + (body_end & 1) # 2-byte alignment
84+
if n_patched > 0:
85+
with open(archive_path, 'wb') as f:
86+
f.write(bytes(out))
87+
return n_patched
88+
89+
90+
def _patch_thin(path):
91+
with open(path, 'rb') as f:
92+
head = f.read(8)
93+
if head[:8] == b'!<arch>\n':
94+
return _patch_static_archive(path)
95+
if len(head) >= 4 and struct.unpack('<I', head[:4])[0] == MH_MAGIC_64:
96+
with open(path, 'rb') as f:
97+
data = f.read()
98+
new_data, patched = _patch_macho_object(data)
99+
if patched:
100+
with open(path, 'wb') as f:
101+
f.write(new_data)
102+
return 1
103+
return 0
104+
105+
106+
def _patch_fat_binary(fat_path):
107+
with open(fat_path, 'rb') as f:
108+
head = f.read(4)
109+
if len(head) < 4:
110+
return 0
111+
magic = struct.unpack('>I', head[:4])[0]
112+
if magic in (FAT_MAGIC, FAT_MAGIC_64):
113+
archs = subprocess.run(
114+
['lipo', '-archs', fat_path],
115+
capture_output=True, text=True, check=True,
116+
).stdout.strip().split()
117+
if 'arm64' not in archs:
118+
return 0
119+
with tempfile.TemporaryDirectory() as td:
120+
arm64_thin = os.path.join(td, 'arm64.bin')
121+
subprocess.run(
122+
['lipo', fat_path, '-thin', 'arm64', '-output', arm64_thin],
123+
check=True,
124+
)
125+
n = _patch_thin(arm64_thin)
126+
if n == 0:
127+
return 0
128+
subprocess.run(
129+
['lipo', fat_path, '-replace', 'arm64', arm64_thin,
130+
'-output', fat_path],
131+
check=True,
132+
)
133+
return n
134+
return _patch_thin(fat_path)
135+
136+
137+
def _find_framework_binary(pod_dir):
138+
fw_dir = os.path.join(pod_dir, 'Frameworks')
139+
if not os.path.isdir(fw_dir):
140+
return None
141+
for name in os.listdir(fw_dir):
142+
if name.endswith('.framework'):
143+
base = name[:-len('.framework')]
144+
binary = os.path.join(fw_dir, name, base)
145+
if os.path.isfile(binary):
146+
return binary
147+
return None
148+
149+
150+
def main(args):
151+
if not args:
152+
print(__doc__, file=sys.stderr)
153+
return 1
154+
total = 0
155+
for path in args:
156+
binary = _find_framework_binary(path)
157+
if not binary:
158+
continue
159+
n = _patch_fat_binary(binary)
160+
if n > 0:
161+
print(f' patched {os.path.basename(binary)}: '
162+
f'{n} object(s) relabeled to iOS Simulator')
163+
total += n
164+
if total > 0:
165+
print(f'[ml_kit] Total Mach-O objects relabeled: {total}')
166+
return 0
167+
168+
169+
if __name__ == '__main__':
170+
sys.exit(main(sys.argv[1:]))

packages/google_mlkit_document_scanner/android/src/main/kotlin/com/google_mlkit_document_scanner/DocumentScanner.kt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,12 @@ class DocumentScanner(
142142

143143
Activity.RESULT_CANCELED -> {
144144
pendingResult?.error(TAG, "Operation cancelled", null)
145+
pendingResult = null
145146
}
146147

147148
else -> {
148149
pendingResult?.error(TAG, "Unknown Error", null)
150+
pendingResult = null
149151
}
150152
}
151153
return true
@@ -175,5 +177,6 @@ class DocumentScanner(
175177
}
176178

177179
pendingResult?.success(resultMap)
180+
pendingResult = null
178181
}
179182
}

0 commit comments

Comments
 (0)