feat: unify gesture planning and multi-touch execution

Author: thymikee

CONTEXT.md

@@ -47,6 +47,8 @@
 - Runner command traits: per-command-type classification for iOS/macOS runner lifecycle behavior, distinct from the public command surface and daemon command registry. The Swift runner traits classify interaction, read-only, and runner-lifecycle axes for XCTest execution; Swift resolves the alert command as read-only only for its `get` action. The TypeScript runner command traits classify daemon-side runner send/recovery policy such as read-only retry routing, readiness probes, and recent-healthy-mutation preflight skips; the TypeScript table is command-type keyed and currently classifies alert as read-only for daemon retry policy. Each side keeps one source of truth keyed by runner command type.
 - Coordinate-first resolved element activation: iOS/macOS runner interaction pattern where a selector or text query resolves the semantic `XCUIElement`, then activation uses the element's resolved center coordinate when a frame is available. This keeps target selection semantic while avoiding `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains focus/remote-driven.
 - Interaction dispatch path: one concrete route an interaction command takes to the device (runtime selector/ref resolution, direct iOS selector, native ref via web clickRef, coordinate, maestro non-hittable fallback). Every path classifies every guarantee in the ADR 0011 registry.
+- Gesture plan: typed, platform-neutral normalization of one- or two-contact gesture intent into bounded pointer trajectories. Contact topology is separate from motion; two-contact intent remains pan/pinch/rotate/transform even when native injection shares one executor. See ADR 0013.
+- Multi-touch geometry: the internal initial span and angle plus centroid translation, scale, and rotation used to build both contact trajectories. Geometry is viewport-aware and fails early when the requested motion cannot fit; it is not a public tuning surface.
 - Guarantee cell: one (dispatch path, guarantee) entry in `src/contracts/interaction-guarantees.ts`, classified as runtime/runner/delegated/inapplicable/waived. Completeness is a compile error; honesty is gate-tested.
 - Owned waiver: a `gap:`-prefixed waived cell carrying a `trackingIssue` URL. Waivers are diffable debt with an owner, never folklore.
 - Parity table: golden JSON fixture under `contracts/fixtures/` consumed by both vitest and the runner's gated Swift tests, so a cross-language rule (e.g. tap-point policy) cannot drift silently. Change the rule only via the table.

android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java

@@ -8,6 +8,7 @@
 import android.view.InputDevice;
 import android.view.MotionEvent;
 import java.nio.charset.StandardCharsets;
+import org.json.JSONArray;
 import org.json.JSONObject;
 
 public final class MultiTouchInstrumentation extends Instrumentation {
@@ -72,13 +73,21 @@ private GestureSpec readSpec(Bundle arguments) throws Exception {
         && !"transform".equals(kind)) {
       throw new IllegalArgumentException("Unsupported kind: " + kind);
     }
-    int x = "swipe".equals(kind) ? payload.getInt("x1") : payload.getInt("x");
-    int y = "swipe".equals(kind) ? payload.getInt("y1") : payload.getInt("y");
+    boolean hasPlannedPointers = payload.has("pointers");
+    int x = hasPlannedPointers
+        ? 0
+        : ("swipe".equals(kind) ? payload.getInt("x1") : payload.getInt("x"));
+    int y = hasPlannedPointers
+        ? 0
+        : ("swipe".equals(kind) ? payload.getInt("y1") : payload.getInt("y"));
     int dx = payload.optInt("dx", 0);
     int dy = payload.optInt("dy", 0);
     int x2 = payload.optInt("x2", x + dx);
     int y2 = payload.optInt("y2", y + dy);
-    int durationMs = clamp(payload.optInt("durationMs", 300), MIN_DURATION_MS, MAX_DURATION_MS);
+    int requestedDurationMs = payload.optInt("durationMs", 300);
+    int durationMs = hasPlannedPointers
+        ? requireInRange(requestedDurationMs, MIN_DURATION_MS, MAX_DURATION_MS, "durationMs")
+        : clamp(requestedDurationMs, MIN_DURATION_MS, MAX_DURATION_MS);
     int radius = clamp(payload.optInt("radius", DEFAULT_RADIUS), MIN_RADIUS, MAX_RADIUS);
     double scale = payload.optDouble("scale", 1.0d);
     double degrees = payload.optDouble("degrees", 0.0d);
@@ -88,10 +97,17 @@ private GestureSpec readSpec(Bundle arguments) throws Exception {
     if (("rotate".equals(kind) || "transform".equals(kind)) && !isFinite(degrees)) {
       throw new IllegalArgumentException("Degrees must be finite");
     }
-    return new GestureSpec(kind, x, y, dx, dy, x2, y2, durationMs, scale, degrees, radius);
+    PlannedPointerPath[] plannedPointers = hasPlannedPointers
+        ? readPlannedPointers(payload.getJSONArray("pointers"), kind, durationMs)
+        : null;
+    return new GestureSpec(
+        kind, x, y, dx, dy, x2, y2, durationMs, scale, degrees, radius, plannedPointers);
   }
 
   private int injectGesture(GestureSpec spec) {
+    if (spec.plannedPointers != null) {
+      return injectPlannedGesture(spec);
+    }
     if ("swipe".equals(spec.kind)) {
       return injectSinglePointerGesture(spec);
     }
@@ -160,6 +176,115 @@ private int injectGesture(GestureSpec spec) {
     }
   }
 
+  private int injectPlannedGesture(GestureSpec spec) {
+    return spec.plannedPointers.length == 1
+        ? injectPlannedSinglePointerGesture(spec)
+        : injectPlannedTwoPointerGesture(spec);
+  }
+
+  private int injectPlannedSinglePointerGesture(GestureSpec spec) {
+    UiAutomation automation = getUiAutomation();
+    long downTime = SystemClock.uptimeMillis();
+    long eventTime = downTime;
+    PointerPair activePointer = plannedPointerPairAt(spec, 0);
+    int count = 0;
+
+    try {
+      injectScheduledEvent(
+          automation,
+          motionEvent(downTime, eventTime, MotionEvent.ACTION_DOWN, activePointer),
+          true);
+      count += 1;
+
+      int lastIndex = spec.plannedPointers[0].samples.length - 1;
+      for (int index = 1; index <= lastIndex; index += 1) {
+        activePointer = plannedPointerPairAt(spec, index);
+        eventTime = downTime + spec.plannedPointers[0].samples[index].offsetMs;
+        injectScheduledEvent(
+            automation,
+            motionEvent(downTime, eventTime, MotionEvent.ACTION_MOVE, activePointer),
+            false);
+        count += 1;
+      }
+
+      injectScheduledEvent(
+          automation,
+          motionEvent(downTime, eventTime, MotionEvent.ACTION_UP, activePointer),
+          true);
+      count += 1;
+      return count;
+    } catch (RuntimeException error) {
+      if (count > 0) {
+        injectCancel(automation, downTime, eventTime + 16, activePointer);
+      }
+      throw error;
+    }
+  }
+
+  private int injectPlannedTwoPointerGesture(GestureSpec spec) {
+    UiAutomation automation = getUiAutomation();
+    long downTime = SystemClock.uptimeMillis();
+    long eventTime = downTime;
+    PointerPair start = plannedPointerPairAt(spec, 0);
+    PointerPair activePointers = start.firstOnly();
+    int count = 0;
+
+    try {
+      injectScheduledEvent(
+          automation,
+          motionEvent(downTime, eventTime, MotionEvent.ACTION_DOWN, activePointers),
+          true);
+      count += 1;
+
+      long firstMoveOffset = spec.plannedPointers[0].samples[1].offsetMs;
+      long secondPointerOffset = Math.max(1, Math.min(8, firstMoveOffset - 1));
+      eventTime = downTime + secondPointerOffset;
+      injectScheduledEvent(
+          automation,
+          motionEvent(
+              downTime,
+              eventTime,
+              MotionEvent.ACTION_POINTER_DOWN | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT),
+              start),
+          true);
+      count += 1;
+      activePointers = start;
+
+      int lastIndex = spec.plannedPointers[0].samples.length - 1;
+      for (int index = 1; index <= lastIndex; index += 1) {
+        activePointers = plannedPointerPairAt(spec, index);
+        eventTime = downTime + spec.plannedPointers[0].samples[index].offsetMs;
+        injectScheduledEvent(
+            automation,
+            motionEvent(downTime, eventTime, MotionEvent.ACTION_MOVE, activePointers),
+            false);
+        count += 1;
+      }
+
+      injectScheduledEvent(
+          automation,
+          motionEvent(
+              downTime,
+              eventTime,
+              MotionEvent.ACTION_POINTER_UP | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT),
+              activePointers),
+          true);
+      count += 1;
+      activePointers = activePointers.firstOnly();
+      injectScheduledEvent(
+          automation,
+          motionEvent(downTime, eventTime + 8, MotionEvent.ACTION_UP, activePointers),
+          true);
+      count += 1;
+      return count;
+    } catch (RuntimeException error) {
+      if (count > 0) {
+        injectCancel(automation, downTime, eventTime + 16, activePointers);
+      }
+      throw error;
+    }
+  }
+
   private int injectSinglePointerGesture(GestureSpec spec) {
     UiAutomation automation = getUiAutomation();
     long downTime = SystemClock.uptimeMillis();
@@ -314,10 +439,89 @@ private static PointerPair pointerPairAt(GestureSpec spec, double t) {
         });
   }
 
+  private static PlannedPointerPath[] readPlannedPointers(
+      JSONArray pointers, String kind, int durationMs) throws Exception {
+    int expectedCount = "swipe".equals(kind) ? 1 : 2;
+    if (pointers.length() != expectedCount) {
+      throw new IllegalArgumentException(
+          "Planned " + kind + " gesture requires exactly " + expectedCount + " pointer paths");
+    }
+    PlannedPointerPath[] result = new PlannedPointerPath[expectedCount];
+    for (int pointerIndex = 0; pointerIndex < expectedCount; pointerIndex += 1) {
+      JSONObject pointer = pointers.getJSONObject(pointerIndex);
+      int pointerId = pointer.getInt("pointerId");
+      if (pointerId != pointerIndex) {
+        throw new IllegalArgumentException("Planned pointer ids must be ordered from 0");
+      }
+      JSONArray samples = pointer.getJSONArray("samples");
+      if (samples.length() < 2) {
+        throw new IllegalArgumentException("Planned pointer path requires at least two samples");
+      }
+      PlannedPointerSample[] parsedSamples = new PlannedPointerSample[samples.length()];
+      long previousOffsetMs = -1;
+      for (int sampleIndex = 0; sampleIndex < samples.length(); sampleIndex += 1) {
+        JSONObject sample = samples.getJSONObject(sampleIndex);
+        double rawOffsetMs = sample.getDouble("offsetMs");
+        double x = sample.getDouble("x");
+        double y = sample.getDouble("y");
+        if (!isFinite(rawOffsetMs) || rawOffsetMs != Math.rint(rawOffsetMs)) {
+          throw new IllegalArgumentException("Planned sample offsetMs must be a finite integer");
+        }
+        if (!isFinite(x) || !isFinite(y)) {
+          throw new IllegalArgumentException("Planned sample coordinates must be finite");
+        }
+        long offsetMs = (long) rawOffsetMs;
+        if (offsetMs <= previousOffsetMs) {
+          throw new IllegalArgumentException("Planned sample offsets must be strictly increasing");
+        }
+        parsedSamples[sampleIndex] = new PlannedPointerSample(offsetMs, (float) x, (float) y);
+        previousOffsetMs = offsetMs;
+      }
+      if (parsedSamples[0].offsetMs != 0
+          || parsedSamples[parsedSamples.length - 1].offsetMs != durationMs) {
+        throw new IllegalArgumentException(
+            "Planned pointer path must start at 0 and end at durationMs");
+      }
+      result[pointerIndex] = new PlannedPointerPath(pointerId, parsedSamples);
+    }
+    if (expectedCount == 2) {
+      PlannedPointerSample[] first = result[0].samples;
+      PlannedPointerSample[] second = result[1].samples;
+      if (first.length != second.length) {
+        throw new IllegalArgumentException("Planned pointer paths must have matching samples");
+      }
+      for (int index = 0; index < first.length; index += 1) {
+        if (first[index].offsetMs != second[index].offsetMs) {
+          throw new IllegalArgumentException("Planned pointer sample offsets must match");
+        }
+      }
+    }
+    return result;
+  }
+
+  private static PointerPair plannedPointerPairAt(GestureSpec spec, int sampleIndex) {
+    int pointerCount = spec.plannedPointers.length;
+    float[] x = new float[pointerCount];
+    float[] y = new float[pointerCount];
+    for (int pointerIndex = 0; pointerIndex < pointerCount; pointerIndex += 1) {
+      PlannedPointerSample sample = spec.plannedPointers[pointerIndex].samples[sampleIndex];
+      x[pointerIndex] = sample.x;
+      y[pointerIndex] = sample.y;
+    }
+    return new PointerPair(x, y);
+  }
+
   private static int clamp(int value, int min, int max) {
     return Math.min(Math.max(value, min), max);
   }
 
+  private static int requireInRange(int value, int min, int max, String field) {
+    if (value < min || value > max) {
+      throw new IllegalArgumentException(field + " must be between " + min + " and " + max);
+    }
+    return value;
+  }
+
   private static boolean isFinite(double value) {
     return !Double.isNaN(value) && !Double.isInfinite(value);
   }
@@ -334,6 +538,7 @@ private static final class GestureSpec {
     final double scale;
     final double degrees;
     final int radius;
+    final PlannedPointerPath[] plannedPointers;
 
     GestureSpec(
         String kind,
@@ -346,7 +551,8 @@ private static final class GestureSpec {
         int durationMs,
         double scale,
         double degrees,
-        int radius) {
+        int radius,
+        PlannedPointerPath[] plannedPointers) {
       this.kind = kind;
       this.x = x;
       this.y = y;
@@ -358,6 +564,29 @@ private static final class GestureSpec {
       this.scale = scale;
       this.degrees = degrees;
       this.radius = radius;
+      this.plannedPointers = plannedPointers;
+    }
+  }
+
+  private static final class PlannedPointerPath {
+    final int pointerId;
+    final PlannedPointerSample[] samples;
+
+    PlannedPointerPath(int pointerId, PlannedPointerSample[] samples) {
+      this.pointerId = pointerId;
+      this.samples = samples;
+    }
+  }
+
+  private static final class PlannedPointerSample {
+    final long offsetMs;
+    final float x;
+    final float y;
+
+    PlannedPointerSample(long offsetMs, float x, float y) {
+      this.offsetMs = offsetMs;
+      this.x = x;
+      this.y = y;
     }
   }
 

apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m

@@ -1,6 +1,7 @@
 #import "RunnerSynthesizedGesture.h"
 
 #import <CoreGraphics/CoreGraphics.h>
+#import <float.h>
 #import <math.h>
 #import <objc/message.h>
 
@@ -13,6 +14,12 @@
 typedef void (*RunnerMsgSendSetInteger)(id, SEL, NSInteger);
 typedef BOOL (*RunnerMsgSendSynthesize)(id, SEL, NSError **);
 
+static double RunnerStagedComponentProgress(double progress, int index, int count) {
+  double start = (double)index / (double)MAX(count, 1);
+  double end = (double)(index + 1) / (double)MAX(count, 1);
+  return MIN(MAX((progress - start) / (end - start), 0.0), 1.0);
+}
+
 typedef struct {
   Class recordClass;
   Class pathClass;
@@ -538,16 +545,30 @@ static CGPoint RunnerPointerPointAt(
   double t,
   double side
 ) {
-  double centerX = x + dx * t;
-  double centerY = y + dy * t;
+  BOOL translates = fabs(dx) > DBL_EPSILON || fabs(dy) > DBL_EPSILON;
+  BOOL scales = fabs(scale - 1.0) > DBL_EPSILON;
+  BOOL rotates = fabs(degrees) > DBL_EPSILON;
+  int componentCount = (translates ? 1 : 0) + (scales ? 1 : 0) + (rotates ? 1 : 0);
+  int componentIndex = 0;
+  double translationProgress = translates
+    ? RunnerStagedComponentProgress(t, componentIndex++, componentCount)
+    : 0.0;
+  double scaleProgress = scales
+    ? RunnerStagedComponentProgress(t, componentIndex++, componentCount)
+    : 0.0;
+  double rotationProgress = rotates
+    ? RunnerStagedComponentProgress(t, componentIndex++, componentCount)
+    : 0.0;
+  double centerX = x + dx * translationProgress;
+  double centerY = y + dy * translationProgress;
   double startRadius = baseRadius / MAX(scale, 1.0);
   double endRadius = baseRadius;
   if (scale < 1.0) {
     startRadius = baseRadius;
     endRadius = baseRadius * scale;
   }
-  double radius = startRadius + (endRadius - startRadius) * t;
-  double angle = (-M_PI_2) + (degrees * M_PI / 180.0) * t;
+  double radius = startRadius + (endRadius - startRadius) * scaleProgress;
+  double angle = (-M_PI_2) + (degrees * M_PI / 180.0) * rotationProgress;
   return CGPointMake(centerX + cos(angle) * radius * side, centerY + sin(angle) * radius * side);
 }