feat: add runtime system and gesture commands

Author: thymikee

COMMAND_OWNERSHIP.md

@@ -69,6 +69,29 @@ Their semantics should live in `agent-device/commands` as they migrate.
   local file inputs remain command-policy gated.
 - `trigger-app-event`: runtime `apps.triggerEvent` implemented with event name
   and JSON payload validation.
+- `back`: runtime `system.back` implemented with typed in-app/system modes.
+- `home`: runtime `system.home` implemented.
+- `rotate`: runtime `system.rotate` implemented with explicit orientation
+  validation.
+- `keyboard`: runtime `system.keyboard` implemented with explicit status/get
+  and dismiss result shapes.
+- `clipboard`: runtime `system.clipboard` implemented with read/write result
+  unions.
+- `settings`: runtime `system.settings` implemented as a typed settings-open
+  primitive.
+- `alert`: runtime `system.alert` implemented with explicit status, handled,
+  and wait result unions.
+- `app-switcher`: runtime `system.appSwitcher` implemented.
+- `focus`: runtime `interactions.focus` implemented for point, ref, and
+  selector targets.
+- `longpress`: runtime `interactions.longPress` implemented for point, ref, and
+  selector targets.
+- `swipe`: runtime `interactions.swipe` implemented with point, ref, selector,
+  and viewport-derived directional starts.
+- `scroll`: runtime `interactions.scroll` implemented with viewport, point, ref,
+  and selector targets.
+- `pinch`: runtime `interactions.pinch` implemented behind the typed backend
+  primitive.
 
 ## Boundary Requirements
 

src/__tests__/runtime-conformance.test.ts

@@ -31,6 +31,19 @@ test('command conformance suites run against a fixture backend', async () => {
   assert.equal(calls.includes('tap'), true);
   assert.equal(calls.includes('fill'), true);
   assert.equal(calls.includes('typeText'), true);
+  assert.equal(calls.includes('focus'), true);
+  assert.equal(calls.includes('longPress'), true);
+  assert.equal(calls.includes('swipe'), true);
+  assert.equal(calls.includes('scroll'), true);
+  assert.equal(calls.includes('pinch'), true);
+  assert.equal(calls.includes('pressBack'), true);
+  assert.equal(calls.includes('pressHome'), true);
+  assert.equal(calls.includes('rotate'), true);
+  assert.equal(calls.includes('setKeyboard'), true);
+  assert.equal(calls.includes('getClipboard'), true);
+  assert.equal(calls.includes('openSettings'), true);
+  assert.equal(calls.includes('handleAlert'), true);
+  assert.equal(calls.includes('openAppSwitcher'), true);
   assert.equal(calls.includes('openApp'), true);
   assert.equal(calls.includes('closeApp'), true);
   assert.equal(calls.includes('listApps'), true);
@@ -78,6 +91,48 @@ function createFixtureBackend(calls: string[]): AgentDeviceBackend {
     typeText: async () => {
       calls.push('typeText');
     },
+    focus: async () => {
+      calls.push('focus');
+    },
+    longPress: async () => {
+      calls.push('longPress');
+    },
+    swipe: async () => {
+      calls.push('swipe');
+    },
+    scroll: async () => {
+      calls.push('scroll');
+    },
+    pinch: async () => {
+      calls.push('pinch');
+    },
+    pressBack: async () => {
+      calls.push('pressBack');
+    },
+    pressHome: async () => {
+      calls.push('pressHome');
+    },
+    rotate: async () => {
+      calls.push('rotate');
+    },
+    setKeyboard: async (_context, options) => {
+      calls.push('setKeyboard');
+      return { action: options.action, visible: false };
+    },
+    getClipboard: async () => {
+      calls.push('getClipboard');
+      return { text: 'copied' };
+    },
+    openSettings: async () => {
+      calls.push('openSettings');
+    },
+    handleAlert: async () => {
+      calls.push('handleAlert');
+      return { kind: 'alertStatus', alert: null };
+    },
+    openAppSwitcher: async () => {
+      calls.push('openAppSwitcher');
+    },
     openApp: async () => {
       calls.push('openApp');
     },

src/__tests__/runtime-interactions.test.ts

@@ -193,6 +193,123 @@ test('runtime typeText validates refs and forwards text to the backend primitive
   );
 });
 
+test('runtime focus and longPress share selector/ref target resolution', async () => {
+  const calls: unknown[] = [];
+  const device = createInteractionDevice(selectorSnapshot(), {
+    focus: async (_context, point) => {
+      calls.push({ command: 'focus', point });
+      return { focused: true };
+    },
+    longPress: async (_context, point, options) => {
+      calls.push({ command: 'longPress', point, durationMs: options?.durationMs });
+    },
+  });
+
+  const focused = await device.interactions.focus(selector('label=Continue'), {
+    session: 'default',
+  });
+  const longPressed = await device.interactions.longPress(ref('@e1'), {
+    session: 'default',
+    durationMs: 750,
+  });
+
+  assert.equal(focused.kind, 'selector');
+  assert.deepEqual(focused.backendResult, { focused: true });
+  assert.equal(longPressed.kind, 'ref');
+  assert.deepEqual(calls, [
+    { command: 'focus', point: { x: 60, y: 40 } },
+    { command: 'longPress', point: { x: 60, y: 40 }, durationMs: 750 },
+  ]);
+});
+
+test('runtime scroll resolves selector targets before calling the backend primitive', async () => {
+  const calls: unknown[] = [];
+  const device = createInteractionDevice(selectorSnapshot(), {
+    scroll: async (_context, target, options) => {
+      calls.push({ target, options });
+      return { scrolled: true };
+    },
+  });
+
+  const selectorResult = await device.interactions.scroll({
+    session: 'default',
+    target: selector('label=Continue'),
+    direction: 'down',
+    pixels: 120,
+  });
+  const viewportResult = await device.interactions.scroll({
+    direction: 'up',
+    amount: 0.5,
+  });
+
+  assert.equal(selectorResult.kind, 'selector');
+  assert.equal(viewportResult.kind, 'viewport');
+  assert.deepEqual(calls, [
+    {
+      target: { kind: 'point', point: { x: 60, y: 40 } },
+      options: { direction: 'down', pixels: 120 },
+    },
+    {
+      target: { kind: 'viewport' },
+      options: { direction: 'up', amount: 0.5 },
+    },
+  ]);
+});
+
+test('runtime swipe supports explicit and viewport-derived targets', async () => {
+  const calls: unknown[] = [];
+  const device = createInteractionDevice(selectorSnapshot(), {
+    swipe: async (_context, from, to, options) => {
+      calls.push({ from, to, durationMs: options?.durationMs });
+    },
+  });
+
+  const explicit = await device.interactions.swipe({
+    from: selector('label=Continue'),
+    to: { x: 200, y: 40 },
+    durationMs: 300,
+    session: 'default',
+  });
+  const directional = await device.interactions.swipe({
+    direction: 'left',
+    distance: 25,
+    session: 'default',
+  });
+
+  assert.deepEqual(explicit.from, { x: 60, y: 40 });
+  assert.deepEqual(directional.from, { x: 60, y: 40 });
+  assert.deepEqual(directional.to, { x: 35, y: 40 });
+  assert.deepEqual(calls, [
+    { from: { x: 60, y: 40 }, to: { x: 200, y: 40 }, durationMs: 300 },
+    { from: { x: 60, y: 40 }, to: { x: 35, y: 40 }, durationMs: undefined },
+  ]);
+});
+
+test('runtime pinch is backend-gated and resolves optional center targets', async () => {
+  const calls: unknown[] = [];
+  const unsupported = createInteractionDevice(selectorSnapshot());
+  await assert.rejects(
+    () => unsupported.interactions.pinch({ scale: 1.2 }),
+    /pinch is not supported/,
+  );
+
+  const device = createInteractionDevice(selectorSnapshot(), {
+    pinch: async (_context, options) => {
+      calls.push(options);
+    },
+  });
+
+  const result = await device.interactions.pinch({
+    scale: 0.8,
+    center: ref('@e1'),
+    session: 'default',
+  });
+
+  assert.equal(result.kind, 'pinch');
+  assert.deepEqual(result.center, { x: 60, y: 40 });
+  assert.deepEqual(calls, [{ scale: 0.8, center: { x: 60, y: 40 } }]);
+});
+
 test('runtime interaction commands are available from the command namespace', async () => {
   const device = createInteractionDevice(selectorSnapshot(), {
     tap: async () => {},
@@ -235,7 +352,20 @@ function fillableSnapshot(): SnapshotState {
 
 function createInteractionDevice(
   snapshot: SnapshotState,
-  overrides: Partial<Pick<AgentDeviceBackend, 'captureSnapshot' | 'tap' | 'fill' | 'typeText'>> & {
+  overrides: Partial<
+    Pick<
+      AgentDeviceBackend,
+      | 'captureSnapshot'
+      | 'tap'
+      | 'fill'
+      | 'typeText'
+      | 'focus'
+      | 'longPress'
+      | 'scroll'
+      | 'swipe'
+      | 'pinch'
+    >
+  > & {
     platform?: AgentDeviceBackend['platform'];
     sessionMetadata?: Record<string, unknown>;
   } = {},
@@ -248,6 +378,13 @@ function createInteractionDevice(
       tap: async (...args) => await overrides.tap?.(...args),
       fill: async (...args) => await overrides.fill?.(...args),
       typeText: async (...args) => await overrides.typeText?.(...args),
+      focus: overrides.focus ? async (...args) => await overrides.focus?.(...args) : undefined,
+      longPress: overrides.longPress
+        ? async (...args) => await overrides.longPress?.(...args)
+        : undefined,
+      scroll: overrides.scroll ? async (...args) => await overrides.scroll?.(...args) : undefined,
+      swipe: overrides.swipe ? async (...args) => await overrides.swipe?.(...args) : undefined,
+      pinch: overrides.pinch ? async (...args) => await overrides.pinch?.(...args) : undefined,
     } satisfies AgentDeviceBackend,
     artifacts: createLocalArtifactAdapter(),
     sessions: createMemorySessionStore([

src/__tests__/runtime-public.test.ts

@@ -43,6 +43,7 @@ const backend = {
   getAppState: async (_context, app: string) => ({ bundleId: app, state: 'foreground' as const }),
   pushFile: async () => {},
   triggerAppEvent: async () => {},
+  pressHome: async () => {},
 } satisfies AgentDeviceBackend;
 
 const artifacts = {
@@ -76,6 +77,7 @@ test('package root exposes command runtime skeleton', async () => {
   assert.equal(device.policy.allowLocalInputPaths, false);
   assert.equal(typeof device.capture.screenshot, 'function');
   assert.equal(typeof device.interactions.click, 'function');
+  assert.equal(typeof device.system.back, 'function');
   assert.equal(typeof device.apps.open, 'function');
   const result = await device.capture.screenshot({});
   assert.equal(result.path, '/tmp/path.png');
@@ -365,11 +367,24 @@ test('public backend, commands, io, and conformance subpaths are importable', ()
   assert.equal(typeof commands.interactions.press, 'function');
   assert.equal(typeof commands.interactions.fill, 'function');
   assert.equal(typeof commands.interactions.typeText, 'function');
+  assert.equal(typeof commands.interactions.focus, 'function');
+  assert.equal(typeof commands.interactions.longPress, 'function');
+  assert.equal(typeof commands.interactions.swipe, 'function');
+  assert.equal(typeof commands.interactions.scroll, 'function');
+  assert.equal(typeof commands.interactions.pinch, 'function');
+  assert.equal(typeof commands.system.back, 'function');
+  assert.equal(typeof commands.system.home, 'function');
+  assert.equal(typeof commands.system.rotate, 'function');
+  assert.equal(typeof commands.system.keyboard, 'function');
+  assert.equal(typeof commands.system.clipboard, 'function');
+  assert.equal(typeof commands.system.settings, 'function');
+  assert.equal(typeof commands.system.alert, 'function');
+  assert.equal(typeof commands.system.appSwitcher, 'function');
   assert.equal(
     commandCatalog.some((entry) => entry.command === 'click' && entry.status === 'implemented'),
     true,
   );
-  assert.equal(commandConformanceSuites.length, 4);
+  assert.equal(commandConformanceSuites.length, 5);
   assert.equal(typeof runCommandConformance, 'function');
   assert.equal(target.name, 'fake');
 });
@@ -421,6 +436,13 @@ test('command router dispatches implemented runtime commands and normalizes erro
   assert.equal(typed.ok, true);
   assert.equal(typed.ok && 'text' in typed.data ? typed.data.text : undefined, 'hello');
 
+  const home = await router.dispatch({
+    command: 'system.home',
+    options: {},
+  });
+  assert.equal(home.ok, true);
+  assert.equal(home.ok && 'kind' in home.data ? home.data.kind : undefined, 'systemHome');
+
   const opened = await router.dispatch({
     command: 'apps.open',
     options: {