Skip to content

Commit 47b981c

Browse files
authored
feat: add gesture command coverage (#576)
* feat: add gesture command coverage * fix: align iOS fling provider fixture * feat: group gesture commands * fix: clarify android gesture support * feat: add android multitouch gestures * fix: address gesture review feedback * refactor: simplify gesture plumbing * fix: keep gesture subcommands internal * fix: update iOS provider pan transcript
1 parent 168de53 commit 47b981c

58 files changed

Lines changed: 3455 additions & 195 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/android.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ jobs:
4343
yes | "$SDKMANAGER" --licenses >/dev/null
4444
"$SDKMANAGER" "platforms;android-36" "build-tools;36.0.0"
4545
46-
- name: Package npm-bundled Android snapshot helper
47-
run: pnpm package:android-snapshot-helper:npm
46+
- name: Package npm-bundled Android helpers
47+
run: |
48+
pnpm package:android-snapshot-helper:npm
49+
pnpm package:android-multitouch-helper:npm
4850
4951
- name: Run Android smoke checks
5052
uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0

.github/workflows/ci.yml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,8 @@ jobs:
104104
- name: Run typecheck
105105
run: pnpm typecheck
106106

107-
android-snapshot-helper:
108-
name: Android Snapshot Helper Package
107+
android-helpers:
108+
name: Android Helper Packages
109109
runs-on: ubuntu-latest
110110
timeout-minutes: 15
111111
steps:
@@ -134,8 +134,10 @@ jobs:
134134
javac --version
135135
java --version
136136
137-
- name: Package npm-bundled Android snapshot helper
138-
run: pnpm package:android-snapshot-helper:npm
137+
- name: Package npm-bundled Android helpers
138+
run: |
139+
pnpm package:android-snapshot-helper:npm
140+
pnpm package:android-multitouch-helper:npm
139141
140142
integration:
141143
name: Integration Tests

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,5 @@ xcuserdata/
3333
.skillgym-results/
3434
android-snapshot-helper/build/
3535
android-snapshot-helper/dist/
36+
android-multitouch-helper/build/
37+
android-multitouch-helper/dist/
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<manifest
2+
xmlns:android="http://schemas.android.com/apk/res/android"
3+
package="com.callstack.agentdevice.multitouchhelper">
4+
<uses-sdk android:minSdkVersion="23" android:targetSdkVersion="36" />
5+
6+
<application
7+
android:debuggable="false"
8+
android:label="Agent Device MultiTouch Helper"
9+
android:testOnly="true"
10+
android:theme="@android:style/Theme.NoDisplay" />
11+
12+
<instrumentation
13+
android:name=".MultiTouchInstrumentation"
14+
android:targetPackage="com.callstack.agentdevice.multitouchhelper"
15+
android:label="Agent Device MultiTouch Helper"
16+
android:functionalTest="true" />
17+
</manifest>
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Android MultiTouch Helper
2+
3+
Small instrumentation APK used to inject Android two-pointer gestures through
4+
`UiAutomation.injectInputEvent`. The helper accepts a compact base64 JSON payload so local ADB,
5+
remote ADB tunnels, and remote providers that allow `adb install -t` plus `am instrument` can use
6+
the same contract.
7+
8+
The helper is separate from `android-snapshot-helper` because the payload and output protocol are
9+
gesture-specific. The install/version/cache lifecycle should stay aligned with the snapshot helper.
10+
11+
## Build
12+
13+
```sh
14+
VERSION="$(node -p 'require("./package.json").version')"
15+
sh ./scripts/build-android-multitouch-helper.sh "$VERSION" .tmp/android-multitouch-helper
16+
```
17+
18+
## Run
19+
20+
```sh
21+
PAYLOAD="$(printf '%s' '{"kind":"transform","x":672,"y":1500,"dx":80,"dy":-40,"scale":1.8,"degrees":35,"durationMs":700}' | base64)"
22+
adb install -r -t ".tmp/android-multitouch-helper/agent-device-android-multitouch-helper-$VERSION.apk"
23+
adb shell am instrument -w \
24+
-e payloadBase64 "$PAYLOAD" \
25+
com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation
26+
```
27+
28+
## Output Contract
29+
30+
The APK emits instrumentation result records using
31+
`agentDeviceProtocol=android-multitouch-helper-v1`.
32+
33+
Successful results include:
34+
35+
- `ok=true`
36+
- `helperApiVersion=1`
37+
- `kind` (`pinch`, `rotate`, or `transform`)
38+
- `injectedEvents`
39+
- `elapsedMs`
40+
41+
Failures return `ok=false`, `errorType`, and `message`.
Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
package com.callstack.agentdevice.multitouchhelper;
2+
3+
import android.app.Instrumentation;
4+
import android.app.UiAutomation;
5+
import android.os.Bundle;
6+
import android.os.SystemClock;
7+
import android.util.Base64;
8+
import android.view.InputDevice;
9+
import android.view.MotionEvent;
10+
import java.nio.charset.StandardCharsets;
11+
import org.json.JSONObject;
12+
13+
public final class MultiTouchInstrumentation extends Instrumentation {
14+
private static final String PROTOCOL = "android-multitouch-helper-v1";
15+
private static final String HELPER_API_VERSION = "1";
16+
private static final int DEFAULT_RADIUS = 160;
17+
private static final int MIN_RADIUS = 24;
18+
private static final int MAX_RADIUS = 1200;
19+
private static final int MIN_DURATION_MS = 16;
20+
private static final int MAX_DURATION_MS = 10_000;
21+
private Bundle arguments;
22+
23+
@Override
24+
public void onCreate(Bundle arguments) {
25+
super.onCreate(arguments);
26+
this.arguments = arguments;
27+
start();
28+
}
29+
30+
@Override
31+
public void onStart() {
32+
super.onStart();
33+
Bundle result = new Bundle();
34+
result.putString("agentDeviceProtocol", PROTOCOL);
35+
result.putString("helperApiVersion", HELPER_API_VERSION);
36+
try {
37+
long startedAtMs = System.currentTimeMillis();
38+
GestureSpec spec = readSpec(arguments);
39+
int injectedEvents = injectGesture(spec);
40+
result.putString("ok", "true");
41+
result.putString("kind", spec.kind);
42+
result.putString("injectedEvents", Integer.toString(injectedEvents));
43+
result.putString("elapsedMs", Long.toString(System.currentTimeMillis() - startedAtMs));
44+
finish(0, result);
45+
} catch (Throwable error) {
46+
result.putString("ok", "false");
47+
result.putString("errorType", error.getClass().getName());
48+
result.putString(
49+
"message",
50+
error.getMessage() == null ? error.getClass().getName() : error.getMessage());
51+
finish(1, result);
52+
}
53+
}
54+
55+
private GestureSpec readSpec(Bundle arguments) throws Exception {
56+
String payloadBase64 = arguments.getString("payloadBase64", "");
57+
if (payloadBase64.isEmpty()) {
58+
throw new IllegalArgumentException("Missing payloadBase64");
59+
}
60+
String json =
61+
new String(Base64.decode(payloadBase64, Base64.DEFAULT), StandardCharsets.UTF_8);
62+
JSONObject payload = new JSONObject(json);
63+
String protocol = payload.optString("protocol", PROTOCOL);
64+
if (!PROTOCOL.equals(protocol)) {
65+
throw new IllegalArgumentException("Unsupported protocol: " + protocol);
66+
}
67+
String kind = payload.getString("kind");
68+
if (!"pinch".equals(kind) && !"rotate".equals(kind) && !"transform".equals(kind)) {
69+
throw new IllegalArgumentException("Unsupported kind: " + kind);
70+
}
71+
int x = payload.getInt("x");
72+
int y = payload.getInt("y");
73+
int dx = payload.optInt("dx", 0);
74+
int dy = payload.optInt("dy", 0);
75+
int durationMs = clamp(payload.optInt("durationMs", 300), MIN_DURATION_MS, MAX_DURATION_MS);
76+
int radius = clamp(payload.optInt("radius", DEFAULT_RADIUS), MIN_RADIUS, MAX_RADIUS);
77+
double scale = payload.optDouble("scale", 1.0d);
78+
double degrees = payload.optDouble("degrees", 0.0d);
79+
if (("pinch".equals(kind) || "transform".equals(kind)) && (!isFinite(scale) || scale <= 0)) {
80+
throw new IllegalArgumentException("Scale must be > 0");
81+
}
82+
if (("rotate".equals(kind) || "transform".equals(kind)) && !isFinite(degrees)) {
83+
throw new IllegalArgumentException("Degrees must be finite");
84+
}
85+
return new GestureSpec(kind, x, y, dx, dy, durationMs, scale, degrees, radius);
86+
}
87+
88+
private int injectGesture(GestureSpec spec) {
89+
UiAutomation automation = getUiAutomation();
90+
long downTime = SystemClock.uptimeMillis();
91+
long eventTime = downTime;
92+
PointerPair start = pointerPairAt(spec, 0);
93+
PointerPair end = pointerPairAt(spec, 1);
94+
int count = 0;
95+
96+
inject(
97+
automation,
98+
motionEvent(downTime, eventTime, MotionEvent.ACTION_DOWN, start.firstOnly()));
99+
count += 1;
100+
eventTime += 8;
101+
inject(
102+
automation,
103+
motionEvent(
104+
downTime,
105+
eventTime,
106+
MotionEvent.ACTION_POINTER_DOWN | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT),
107+
start));
108+
count += 1;
109+
110+
int frameCount = Math.max(3, Math.round(spec.durationMs / 16.0f));
111+
for (int index = 1; index < frameCount; index += 1) {
112+
double t = (double) index / (double) frameCount;
113+
PointerPair frame = pointerPairAt(spec, t);
114+
eventTime = downTime + Math.round(spec.durationMs * t);
115+
inject(automation, motionEvent(downTime, eventTime, MotionEvent.ACTION_MOVE, frame));
116+
count += 1;
117+
}
118+
119+
eventTime = downTime + spec.durationMs;
120+
inject(
121+
automation,
122+
motionEvent(
123+
downTime,
124+
eventTime,
125+
MotionEvent.ACTION_POINTER_UP | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT),
126+
end));
127+
count += 1;
128+
inject(
129+
automation,
130+
motionEvent(downTime, eventTime + 8, MotionEvent.ACTION_UP, end.firstOnly()));
131+
count += 1;
132+
return count;
133+
}
134+
135+
private static void inject(UiAutomation automation, MotionEvent event) {
136+
try {
137+
if (!automation.injectInputEvent(event, true)) {
138+
throw new IllegalStateException("injectInputEvent returned false");
139+
}
140+
} finally {
141+
event.recycle();
142+
}
143+
}
144+
145+
private static MotionEvent motionEvent(long downTime, long eventTime, int action, PointerPair pair) {
146+
MotionEvent.PointerProperties[] properties =
147+
new MotionEvent.PointerProperties[pair.pointerCount];
148+
MotionEvent.PointerCoords[] coords = new MotionEvent.PointerCoords[pair.pointerCount];
149+
for (int index = 0; index < pair.pointerCount; index += 1) {
150+
properties[index] = new MotionEvent.PointerProperties();
151+
properties[index].id = index;
152+
properties[index].toolType = MotionEvent.TOOL_TYPE_FINGER;
153+
coords[index] = new MotionEvent.PointerCoords();
154+
coords[index].x = pair.x[index];
155+
coords[index].y = pair.y[index];
156+
coords[index].pressure = 1.0f;
157+
coords[index].size = 1.0f;
158+
}
159+
MotionEvent event =
160+
MotionEvent.obtain(
161+
downTime,
162+
eventTime,
163+
action,
164+
pair.pointerCount,
165+
properties,
166+
coords,
167+
0,
168+
0,
169+
1.0f,
170+
1.0f,
171+
0,
172+
0,
173+
InputDevice.SOURCE_TOUCHSCREEN,
174+
0);
175+
event.setSource(InputDevice.SOURCE_TOUCHSCREEN);
176+
return event;
177+
}
178+
179+
private static PointerPair pointerPairAt(GestureSpec spec, double t) {
180+
if ("pinch".equals(spec.kind)) {
181+
double startRadius = spec.radius / Math.max(spec.scale, 1.0d);
182+
double endRadius = spec.radius;
183+
if (spec.scale < 1.0d) {
184+
startRadius = spec.radius;
185+
endRadius = spec.radius * spec.scale;
186+
}
187+
double radius = startRadius + (endRadius - startRadius) * t;
188+
return new PointerPair(
189+
new float[] {(float) (spec.x - radius), (float) (spec.x + radius)},
190+
new float[] {(float) spec.y, (float) spec.y});
191+
}
192+
double centerX = spec.x;
193+
double centerY = spec.y;
194+
double radius = spec.radius;
195+
if ("transform".equals(spec.kind)) {
196+
centerX = spec.x + spec.dx * t;
197+
centerY = spec.y + spec.dy * t;
198+
double startRadius = spec.radius / Math.max(spec.scale, 1.0d);
199+
double endRadius = spec.radius;
200+
if (spec.scale < 1.0d) {
201+
startRadius = spec.radius;
202+
endRadius = spec.radius * spec.scale;
203+
}
204+
radius = startRadius + (endRadius - startRadius) * t;
205+
}
206+
double angle = Math.toRadians(-90 + spec.degrees * t);
207+
return new PointerPair(
208+
new float[] {
209+
(float) (centerX + Math.cos(angle) * radius),
210+
(float) (centerX - Math.cos(angle) * radius)
211+
},
212+
new float[] {
213+
(float) (centerY + Math.sin(angle) * radius),
214+
(float) (centerY - Math.sin(angle) * radius)
215+
});
216+
}
217+
218+
private static int clamp(int value, int min, int max) {
219+
return Math.min(Math.max(value, min), max);
220+
}
221+
222+
private static boolean isFinite(double value) {
223+
return !Double.isNaN(value) && !Double.isInfinite(value);
224+
}
225+
226+
private static final class GestureSpec {
227+
final String kind;
228+
final int x;
229+
final int y;
230+
final int dx;
231+
final int dy;
232+
final int durationMs;
233+
final double scale;
234+
final double degrees;
235+
final int radius;
236+
237+
GestureSpec(
238+
String kind,
239+
int x,
240+
int y,
241+
int dx,
242+
int dy,
243+
int durationMs,
244+
double scale,
245+
double degrees,
246+
int radius) {
247+
this.kind = kind;
248+
this.x = x;
249+
this.y = y;
250+
this.dx = dx;
251+
this.dy = dy;
252+
this.durationMs = durationMs;
253+
this.scale = scale;
254+
this.degrees = degrees;
255+
this.radius = radius;
256+
}
257+
}
258+
259+
private static final class PointerPair {
260+
final int pointerCount;
261+
final float[] x;
262+
final float[] y;
263+
264+
PointerPair(float[] x, float[] y) {
265+
this.pointerCount = x.length;
266+
this.x = x;
267+
this.y = y;
268+
}
269+
270+
PointerPair firstOnly() {
271+
return new PointerPair(
272+
new float[] {x[0]},
273+
new float[] {y[0]});
274+
}
275+
}
276+
}

examples/test-app/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,11 @@ pnpm test-app:replay:android
8888

8989
These run the `.ad` replay suite in `examples/test-app/replays`.
9090

91+
`gesture-lab.ad` verifies `gesture pan`, `gesture fling`, `gesture pinch`, and
92+
`gesture rotate` against the gesture metrics rendered by the Home screen on iOS
93+
and Android. Android and iOS simulator sessions also support `gesture transform`
94+
for a combined pan/zoom/rotate gesture.
95+
9196
To target a specific iOS simulator or an installed Expo development build, run the
9297
underlying command directly so global flags stay before replay inputs:
9398

0 commit comments

Comments
 (0)