Skip to content

Commit a172209

Browse files
NorbertKlockiewiczNorbertKlockiewicz
committed
refactor: fixes after rebase + requested changes
1 parent 7bea029 commit a172209

16 files changed

Lines changed: 169 additions & 142 deletions

File tree

‎apps/computer-vision/components/vision_camera/tasks/InstanceSegmentationTask.tsx‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ type Props = TaskProps & { activeModel: InstSegModelId };
2727
export default function InstanceSegmentationTask({
2828
activeModel,
2929
canvasSize,
30-
cameraPosition,
30+
cameraPositionSync,
3131
frameKillSwitch,
3232
onFrameOutputChange,
3333
onReadyChange,
@@ -145,13 +145,7 @@ export default function InstanceSegmentationTask({
145145
const offsetY = (canvasSize.height - imageSize.height * scale) / 2;
146146

147147
return (
148-
<View
149-
style={[
150-
StyleSheet.absoluteFill,
151-
cameraPosition === 'front' && { transform: [{ scaleX: -1 }] },
152-
]}
153-
pointerEvents="none"
154-
>
148+
<View style={StyleSheet.absoluteFill} pointerEvents="none">
155149
{/* Render masks */}
156150
<Canvas style={StyleSheet.absoluteFill} pointerEvents="none">
157151
{instances.map((inst, i) => {
@@ -197,7 +191,6 @@ export default function InstanceSegmentationTask({
197191
style={[
198192
styles.bboxLabel,
199193
{ backgroundColor: labelColorBg(label) },
200-
cameraPosition === 'front' && { transform: [{ scaleX: -1 }] },
201194
]}
202195
>
203196
<Text style={styles.bboxLabelText}>

‎packages/react-native-executorch/common/rnexecutorch/models/VisionModel.cpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,4 @@ cv::Mat VisionModel::extractFromPixels(const JSTensorViewIn &tensorView) const {
5151
return ::rnexecutorch::utils::pixelsToMat(tensorView);
5252
}
5353

54-
utils::FrameOrientation VisionModel::extractFrameOrientation(
55-
jsi::Runtime &runtime, const jsi::Value &frameData) const {
56-
return ::rnexecutorch::utils::readFrameOrientation(runtime, frameData);
57-
}
58-
5954
} // namespace rnexecutorch::models

‎packages/react-native-executorch/common/rnexecutorch/models/VisionModel.h‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -144,16 +144,6 @@ class VisionModel : public BaseModel {
144144
* @endcode
145145
*/
146146
cv::Mat extractFromPixels(const JSTensorViewIn &tensorView) const;
147-
148-
/**
149-
* @brief Read orientation metadata from JSI frameData object.
150-
*
151-
* Reads orientation and isMirrored.
152-
* Falls back to "up"/false if fields are absent (e.g. when
153-
* enablePhysicalBufferRotation is used — transform will then be a no-op).
154-
*/
155-
utils::FrameOrientation extractFrameOrientation(
156-
jsi::Runtime &runtime, const jsi::Value &frameData) const;
157147
};
158148

159149
} // namespace models

‎packages/react-native-executorch/common/rnexecutorch/models/object_detection/ObjectDetection.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,12 @@ std::vector<types::Detection>
139139
ObjectDetection::generateFromFrame(jsi::Runtime &runtime,
140140
const jsi::Value &frameData,
141141
double detectionThreshold) {
142-
auto orient = extractFrameOrientation(runtime, frameData);
142+
auto orient = ::rnexecutorch::utils::readFrameOrientation(runtime, frameData);
143143
cv::Mat frame = extractFromFrame(runtime, frameData);
144144
cv::Mat rotated = ::rnexecutorch::utils::rotateFrameForModel(frame, orient);
145145
auto detections = runInference(rotated, detectionThreshold);
146146
for (auto &det : detections) {
147-
::rnexecutorch::utils::inverseRotateBbox(
148-
det.x1, det.y1, det.x2, det.y2, orient, rotated.cols, rotated.rows);
147+
::rnexecutorch::utils::inverseRotateBbox(det.bbox, orient, rotated.size());
149148
}
150149
return detections;
151150
}

‎packages/react-native-executorch/common/rnexecutorch/models/ocr/OCR.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <rnexecutorch/data_processing/ImageProcessing.h>
66
#include <rnexecutorch/models/ocr/Constants.h>
77
#include <rnexecutorch/utils/FrameProcessor.h>
8+
#include <rnexecutorch/utils/FrameTransform.h>
89

910
namespace rnexecutorch::models::ocr {
1011
OCR::OCR(const std::string &detectorSource, const std::string &recognizerSource,
@@ -67,7 +68,7 @@ OCR::generateFromFrame(jsi::Runtime &runtime, const jsi::Value &frameData) {
6768
auto detections = runInference(rotated);
6869
for (auto &det : detections) {
6970
::rnexecutorch::utils::inverseRotatePoints(det.bbox, orient,
70-
rotated.cols, rotated.rows);
71+
rotated.size());
7172
}
7273
return detections;
7374
}

‎packages/react-native-executorch/common/rnexecutorch/models/semantic_segmentation/BaseSemanticSegmentation.cpp‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <rnexecutorch/Log.h>
77
#include <rnexecutorch/data_processing/ImageProcessing.h>
88
#include <rnexecutorch/models/BaseModel.h>
9+
#include <rnexecutorch/utils/FrameProcessor.h>
910
#include <rnexecutorch/utils/FrameTransform.h>
1011

1112
namespace rnexecutorch::models::semantic_segmentation {
@@ -96,35 +97,36 @@ semantic_segmentation::SegmentationResult
9697
BaseSemanticSegmentation::generateFromFrame(
9798
jsi::Runtime &runtime, const jsi::Value &frameData,
9899
std::set<std::string, std::less<>> classesOfInterest, bool resize) {
99-
auto orient = extractFrameOrientation(runtime, frameData);
100+
auto orient = ::rnexecutorch::utils::readFrameOrientation(runtime, frameData);
100101
cv::Mat frame = extractFromFrame(runtime, frameData);
101102
cv::Mat rotated = utils::rotateFrameForModel(frame, orient);
102103
// Always run inference without resize — rotate first, then resize.
103104
auto result = runInference(rotated, rotated.size(), classesOfInterest, false);
104105

105106
const cv::Size outputSize = modelInputSize();
106107
// JS reads maskW=frame.height, maskH=frame.width (sensor-native swap).
107-
const cv::Size screenSize(frame.rows, frame.cols);
108+
const cv::Size frameSize = frame.size();
108109

109-
auto inverseAndResize = [&](std::shared_ptr<OwningArrayBuffer> &buf,
110-
int32_t cvType, int32_t interpFlag) {
110+
auto inverseAndResize = [&orient, &frameSize, &outputSize,
111+
resize](std::shared_ptr<OwningArrayBuffer> &buf,
112+
int32_t cvType, int32_t interpFlag) {
111113
cv::Mat m(outputSize, cvType, buf->data());
112114
cv::Mat inv = utils::inverseRotateMat(m, orient);
113-
if (resize && inv.size() != screenSize) {
114-
cv::resize(inv, inv, screenSize, 0, 0, interpFlag);
115+
if (resize && inv.size() != frameSize) {
116+
cv::resize(inv, inv, frameSize, 0, 0, interpFlag);
115117
}
116118
buf = std::make_shared<OwningArrayBuffer>(
117119
inv.data, static_cast<size_t>(inv.total() * inv.elemSize()));
118-
return inv;
119120
};
120121

121-
if (result.argmax && outputSize.area() > 0) {
122-
inverseAndResize(result.argmax, CV_32SC1, cv::INTER_NEAREST);
123-
}
124-
125-
if (result.classBuffers && outputSize.area() > 0) {
126-
for (auto &[label, buf] : *result.classBuffers) {
127-
inverseAndResize(buf, CV_32FC1, cv::INTER_LINEAR);
122+
if (outputSize.area() > 0) {
123+
if (result.argmax) {
124+
inverseAndResize(result.argmax, CV_32SC1, cv::INTER_NEAREST);
125+
}
126+
if (result.classBuffers) {
127+
for (auto &[label, buf] : *result.classBuffers) {
128+
inverseAndResize(buf, CV_32FC1, cv::INTER_LINEAR);
129+
}
128130
}
129131
}
130132

‎packages/react-native-executorch/common/rnexecutorch/models/style_transfer/StyleTransfer.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "StyleTransfer.h"
22

33
#include <rnexecutorch/data_processing/ImageProcessing.h>
4+
#include <rnexecutorch/utils/FrameProcessor.h>
45
#include <rnexecutorch/utils/FrameTransform.h>
56

67
#include <executorch/extension/tensor/tensor.h>
@@ -82,7 +83,7 @@ StyleTransferResult StyleTransfer::generateFromString(std::string imageSource,
8283

8384
PixelDataResult StyleTransfer::generateFromFrame(jsi::Runtime &runtime,
8485
const jsi::Value &frameData) {
85-
auto orient = extractFrameOrientation(runtime, frameData);
86+
auto orient = ::rnexecutorch::utils::readFrameOrientation(runtime, frameData);
8687
cv::Mat frame = extractFromFrame(runtime, frameData);
8788
cv::Mat rotated = utils::rotateFrameForModel(frame, orient);
8889
cv::Mat output = runInference(rotated, modelInputSize());

‎packages/react-native-executorch/common/rnexecutorch/models/vertical_ocr/VerticalOCR.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <rnexecutorch/models/ocr/Constants.h>
77
#include <rnexecutorch/models/ocr/Types.h>
88
#include <rnexecutorch/utils/FrameProcessor.h>
9+
#include <rnexecutorch/utils/FrameTransform.h>
910
#include <tuple>
1011

1112
namespace rnexecutorch::models::ocr {
@@ -71,7 +72,7 @@ VerticalOCR::generateFromFrame(jsi::Runtime &runtime,
7172
auto detections = runInference(rotated);
7273
for (auto &det : detections) {
7374
::rnexecutorch::utils::inverseRotatePoints(det.bbox, orient,
74-
rotated.cols, rotated.rows);
75+
rotated.size());
7576
}
7677
return detections;
7778
}

‎packages/react-native-executorch/common/rnexecutorch/tests/unit/FrameTransformTest.cpp‎

Lines changed: 40 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
#include <gtest/gtest.h>
22
#include <opencv2/opencv.hpp>
33
#include <rnexecutorch/utils/FrameTransform.h>
4+
#include <rnexecutorch/utils/computer_vision/Types.h>
45

56
using namespace rnexecutorch::utils;
7+
using rnexecutorch::utils::computer_vision::BBox;
68

79
static FrameOrientation makeOrient(const std::string &o, bool mirrored) {
810
return {orientationFromString(o), mirrored};
911
}
1012

1113
// ============================================================================
12-
// rotateFrameForModel — rotates sensor-native frame so model sees upright image.
14+
// rotateFrameForModel — rotates sensor-native frame so model sees upright
15+
// image.
1316
//
1417
// "up" (landscape-left) → no rotation
1518
// "left" (portrait upright) → CW
@@ -215,7 +218,8 @@ TEST(InverseRotateMat, DoesNotModifyInput) {
215218
// rW/rH = rotated frame dimensions (after rotateFrameForModel).
216219
// On Android (no __APPLE__), isMirrored is ignored.
217220
//
218-
// Formulas (same as inverseRotatePoints per-corner, but preserves x1<=x2, y1<=y2):
221+
// Formulas (same as inverseRotatePoints per-corner, but preserves x1<=x2,
222+
// y1<=y2):
219223
// "up" → CW: nx1=h-y2, ny1=x1, nx2=h-y1, ny2=x2
220224
// "right" → 180°: nx1=w-x2, ny1=h-y2, nx2=w-x1, ny2=h-y1
221225
// "down" → CCW: nx1=y1, ny1=w-x2, nx2=y2, ny2=w-x1
@@ -224,53 +228,53 @@ TEST(InverseRotateMat, DoesNotModifyInput) {
224228

225229
// "left" → no-op. Box unchanged.
226230
TEST(InverseRotateBbox, Left_NoOp) {
227-
float x1 = 10, y1 = 20, x2 = 100, y2 = 200;
228-
inverseRotateBbox(x1, y1, x2, y2, makeOrient("left", false), 640, 480);
229-
EXPECT_FLOAT_EQ(x1, 10);
230-
EXPECT_FLOAT_EQ(y1, 20);
231-
EXPECT_FLOAT_EQ(x2, 100);
232-
EXPECT_FLOAT_EQ(y2, 200);
231+
BBox bbox{10, 20, 100, 200};
232+
inverseRotateBbox(bbox, makeOrient("left", false), {640, 480});
233+
EXPECT_FLOAT_EQ(bbox.x1, 10);
234+
EXPECT_FLOAT_EQ(bbox.y1, 20);
235+
EXPECT_FLOAT_EQ(bbox.x2, 100);
236+
EXPECT_FLOAT_EQ(bbox.y2, 200);
233237
}
234238

235239
// "up" → CW. rW=640, rH=480. Box (10,20)-(100,200):
236240
// nx1=480-200=280, ny1=10, nx2=480-20=460, ny2=100
237241
TEST(InverseRotateBbox, Up_CW) {
238-
float x1 = 10, y1 = 20, x2 = 100, y2 = 200;
239-
inverseRotateBbox(x1, y1, x2, y2, makeOrient("up", false), 640, 480);
240-
EXPECT_FLOAT_EQ(x1, 280);
241-
EXPECT_FLOAT_EQ(y1, 10);
242-
EXPECT_FLOAT_EQ(x2, 460);
243-
EXPECT_FLOAT_EQ(y2, 100);
242+
BBox bbox{10, 20, 100, 200};
243+
inverseRotateBbox(bbox, makeOrient("up", false), {640, 480});
244+
EXPECT_FLOAT_EQ(bbox.x1, 280);
245+
EXPECT_FLOAT_EQ(bbox.y1, 10);
246+
EXPECT_FLOAT_EQ(bbox.x2, 460);
247+
EXPECT_FLOAT_EQ(bbox.y2, 100);
244248
}
245249

246250
// "right" → 180°. rW=480, rH=640. Box (10,20)-(100,200):
247251
// nx1=480-100=380, ny1=640-200=440, nx2=480-10=470, ny2=640-20=620
248252
TEST(InverseRotateBbox, Right_180) {
249-
float x1 = 10, y1 = 20, x2 = 100, y2 = 200;
250-
inverseRotateBbox(x1, y1, x2, y2, makeOrient("right", false), 480, 640);
251-
EXPECT_FLOAT_EQ(x1, 380);
252-
EXPECT_FLOAT_EQ(y1, 440);
253-
EXPECT_FLOAT_EQ(x2, 470);
254-
EXPECT_FLOAT_EQ(y2, 620);
253+
BBox bbox{10, 20, 100, 200};
254+
inverseRotateBbox(bbox, makeOrient("right", false), {480, 640});
255+
EXPECT_FLOAT_EQ(bbox.x1, 380);
256+
EXPECT_FLOAT_EQ(bbox.y1, 440);
257+
EXPECT_FLOAT_EQ(bbox.x2, 470);
258+
EXPECT_FLOAT_EQ(bbox.y2, 620);
255259
}
256260

257261
// "down" → CCW. rW=640, rH=480. Box (10,20)-(100,200):
258262
// nx1=20, ny1=640-100=540, nx2=200, ny2=640-10=630
259263
TEST(InverseRotateBbox, Down_CCW) {
260-
float x1 = 10, y1 = 20, x2 = 100, y2 = 200;
261-
inverseRotateBbox(x1, y1, x2, y2, makeOrient("down", false), 640, 480);
262-
EXPECT_FLOAT_EQ(x1, 20);
263-
EXPECT_FLOAT_EQ(y1, 540);
264-
EXPECT_FLOAT_EQ(x2, 200);
265-
EXPECT_FLOAT_EQ(y2, 630);
264+
BBox bbox{10, 20, 100, 200};
265+
inverseRotateBbox(bbox, makeOrient("down", false), {640, 480});
266+
EXPECT_FLOAT_EQ(bbox.x1, 20);
267+
EXPECT_FLOAT_EQ(bbox.y1, 540);
268+
EXPECT_FLOAT_EQ(bbox.x2, 200);
269+
EXPECT_FLOAT_EQ(bbox.y2, 630);
266270
}
267271

268272
// Guarantees x1<=x2 and y1<=y2 after transform.
269273
TEST(InverseRotateBbox, OutputOrdered) {
270-
float x1 = 50, y1 = 50, x2 = 150, y2 = 250;
271-
inverseRotateBbox(x1, y1, x2, y2, makeOrient("up", false), 640, 480);
272-
EXPECT_LE(x1, x2);
273-
EXPECT_LE(y1, y2);
274+
BBox bbox{50, 50, 150, 250};
275+
inverseRotateBbox(bbox, makeOrient("up", false), {640, 480});
276+
EXPECT_LE(bbox.x1, bbox.x2);
277+
EXPECT_LE(bbox.y1, bbox.y2);
274278
}
275279

276280
// ============================================================================
@@ -288,7 +292,7 @@ struct Pt {
288292
// "left" → no-op. Points unchanged.
289293
TEST(InverseRotatePoints, Left_NoOp) {
290294
std::array<Pt, 4> pts = {{{10, 20}, {30, 40}, {50, 60}, {70, 80}}};
291-
inverseRotatePoints(pts, makeOrient("left", false), 640, 480);
295+
inverseRotatePoints(pts, makeOrient("left", false), {640, 480});
292296
EXPECT_FLOAT_EQ(pts[0].x, 10);
293297
EXPECT_FLOAT_EQ(pts[0].y, 20);
294298
EXPECT_FLOAT_EQ(pts[1].x, 30);
@@ -302,7 +306,7 @@ TEST(InverseRotatePoints, Left_NoOp) {
302306
// "up" → CW per point. rW=640, rH=480. pt(10,20): nx=480-20=460, ny=10.
303307
TEST(InverseRotatePoints, Up_CW) {
304308
std::array<Pt, 4> pts = {{{10, 20}, {30, 40}, {50, 60}, {70, 80}}};
305-
inverseRotatePoints(pts, makeOrient("up", false), 640, 480);
309+
inverseRotatePoints(pts, makeOrient("up", false), {640, 480});
306310
EXPECT_FLOAT_EQ(pts[0].x, 460);
307311
EXPECT_FLOAT_EQ(pts[0].y, 10);
308312
EXPECT_FLOAT_EQ(pts[1].x, 440);
@@ -313,10 +317,11 @@ TEST(InverseRotatePoints, Up_CW) {
313317
EXPECT_FLOAT_EQ(pts[3].y, 70);
314318
}
315319

316-
// "right" → 180° per point. rW=480, rH=640. pt(10,20): nx=480-10=470, ny=640-20=620.
320+
// "right" → 180° per point. rW=480, rH=640. pt(10,20): nx=480-10=470,
321+
// ny=640-20=620.
317322
TEST(InverseRotatePoints, Right_180) {
318323
std::array<Pt, 4> pts = {{{10, 20}, {30, 40}, {50, 60}, {70, 80}}};
319-
inverseRotatePoints(pts, makeOrient("right", false), 480, 640);
324+
inverseRotatePoints(pts, makeOrient("right", false), {480, 640});
320325
EXPECT_FLOAT_EQ(pts[0].x, 470);
321326
EXPECT_FLOAT_EQ(pts[0].y, 620);
322327
EXPECT_FLOAT_EQ(pts[1].x, 450);
@@ -330,7 +335,7 @@ TEST(InverseRotatePoints, Right_180) {
330335
// "down" → CCW per point. rW=640, rH=480. pt(10,20): nx=20, ny=640-10=630.
331336
TEST(InverseRotatePoints, Down_CCW) {
332337
std::array<Pt, 4> pts = {{{10, 20}, {30, 40}, {50, 60}, {70, 80}}};
333-
inverseRotatePoints(pts, makeOrient("down", false), 640, 480);
338+
inverseRotatePoints(pts, makeOrient("down", false), {640, 480});
334339
EXPECT_FLOAT_EQ(pts[0].x, 20);
335340
EXPECT_FLOAT_EQ(pts[0].y, 630);
336341
EXPECT_FLOAT_EQ(pts[1].x, 40);

‎packages/react-native-executorch/common/rnexecutorch/utils/FrameProcessor.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,17 @@ FrameOrientation readFrameOrientation(jsi::Runtime &runtime,
3838
std::string orientStr = "up";
3939
if (obj.hasProperty(runtime, "orientation")) {
4040
auto val = obj.getProperty(runtime, "orientation");
41-
if (val.isString())
41+
if (val.isString()) {
4242
orientStr = val.getString(runtime).utf8(runtime);
43+
}
4344
}
4445

4546
bool isMirrored = false;
4647
if (obj.hasProperty(runtime, "isMirrored")) {
4748
auto val = obj.getProperty(runtime, "isMirrored");
48-
if (val.isBool())
49+
if (val.isBool()) {
4950
isMirrored = val.getBool();
51+
}
5052
}
5153

5254
return {orientationFromString(orientStr), isMirrored};

0 commit comments

Comments
 (0)