feat: enable tvOS runner interactions

Author: thymikee

README.md

@@ -196,7 +196,7 @@ Efficient snapshot usage:
 
 Flags:
 - `--version, -V` print version and exit
-- `--platform ios|android`
+- `--platform ios|android|apple` (`apple` aliases the iOS/tvOS backend)
 - `--target mobile|tv` select device class within platform (requires `--platform`; for example AndroidTV/tvOS)
 - `--device <name>`
 - `--udid <udid>` (iOS)
@@ -218,10 +218,10 @@ Flags:
 - `--max-steps <n>` batch: max allowed steps per request
 
 TV targets:
-- Use `--target tv` together with `--platform ios|android`.
+- Use `--target tv` together with `--platform ios|android|apple`.
 - AndroidTV app launch/app listing use TV launcher discovery (`LEANBACK_LAUNCHER`) and fallback component resolution when needed.
-- tvOS currently supports non-runner flows only: `open`, `close`, `apps`, `screenshot`, `logs`, `reinstall`, `boot`.
-- tvOS runner-driven interaction/snapshot commands remain unsupported: `snapshot`, `wait`, `press`, `fill`, `get`, `scroll`, `back`, `home`, `app-switcher`, `record`.
+- tvOS uses the same runner-driven interaction/snapshot flow as iOS (`snapshot`, `wait`, `press`, `fill`, `get`, `scroll`, `back`, `home`, `app-switcher`, `record`, and related selector flows).
+- tvOS back/home/app-switcher use Siri Remote semantics in the runner (`menu`, `home`, double-home).
 
 Examples:
 - `agent-device open YouTube --platform android --target tv`
@@ -259,7 +259,7 @@ Sessions:
 - On iOS, `appstate` is session-scoped and requires an active session on the target device.
 
 Navigation helpers:
-- `boot --platform ios|android` ensures the target is ready without launching an app.
+- `boot --platform ios|android|apple` ensures the target is ready without launching an app.
 - Use `boot` mainly when starting a new session and `open` fails because no booted simulator/emulator is available.
 - `open [app|url] [url]` already boots/activates the selected target when needed.
 - `reinstall <app> <path>` uninstalls and installs the app binary in one command (Android + iOS simulator/device).
@@ -367,7 +367,7 @@ Boot diagnostics:
 - Boot failures include normalized reason codes in `error.details.reason` (JSON mode) and verbose logs.
 - Reason codes: `IOS_BOOT_TIMEOUT`, `IOS_RUNNER_CONNECT_TIMEOUT`, `ANDROID_BOOT_TIMEOUT`, `ADB_TRANSPORT_UNAVAILABLE`, `CI_RESOURCE_STARVATION_SUSPECTED`, `BOOT_COMMAND_FAILED`, `UNKNOWN`.
 - Android boot waits fail fast for permission/tooling issues and do not always collapse into timeout errors.
-- Use `agent-device boot --platform ios|android` when starting a new session only if `open` cannot find/connect to an available target.
+- Use `agent-device boot --platform ios|android|apple` when starting a new session only if `open` cannot find/connect to an available target.
 - `--debug` captures retry telemetry in diagnostics logs.
 - Set `AGENT_DEVICE_RETRY_LOGS=1` to also print retry telemetry directly to stderr (ad-hoc troubleshooting).
 
@@ -384,8 +384,7 @@ Diagnostics files:
 ## iOS notes
 - Core runner commands: `snapshot`, `wait`, `click`, `fill`, `get`, `is`, `find`, `press`, `longpress`, `focus`, `type`, `scroll`, `scrollintoview`, `back`, `home`, `app-switcher`.
 - Simulator-only commands: `alert`, `pinch`, `settings`.
-- tvOS targets are selectable (`--platform ios --target tv`) for non-runner flows (`open`, `close`, `apps`, `screenshot`, `logs`, `reinstall`, `boot`).
-- tvOS runner-driven interaction commands are currently unsupported (`snapshot`, `wait`, `press`, `fill`, `get`, `scroll`, `back`, `home`, `app-switcher`, `record`).
+- tvOS targets are selectable (`--platform ios --target tv` or `--platform apple --target tv`) and support runner-driven interaction/snapshot commands.
 - `record` supports iOS simulators and physical iOS devices.
   - iOS simulator recording uses native `simctl io ... recordVideo`.
   - Physical iOS device recording is runner-based and built from repeated `XCUIScreen.main.screenshot()` frames (no native video stream/audio capture).

ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj

@@ -258,7 +258,7 @@
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				ONLY_ACTIVE_ARCH = YES;
-				SDKROOT = iphoneos;
+				SDKROOT = auto;
 				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)";
 				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
 			};
@@ -315,7 +315,7 @@
 				LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				MTL_FAST_MATH = YES;
-				SDKROOT = iphoneos;
+				SDKROOT = auto;
 				SWIFT_COMPILATION_MODE = wholemodule;
 				VALIDATE_PRODUCT = YES;
 			};
@@ -350,7 +350,9 @@
 				SWIFT_EMIT_LOC_STRINGS = YES;
 				SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
 				SWIFT_VERSION = 5.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator appletvos appletvsimulator";
+				TARGETED_DEVICE_FAMILY = "1,2,3";
+				TVOS_DEPLOYMENT_TARGET = 15.6;
 			};
 			name = Debug;
 		};
@@ -383,7 +385,9 @@
 				SWIFT_EMIT_LOC_STRINGS = YES;
 				SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
 				SWIFT_VERSION = 5.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator appletvos appletvsimulator";
+				TARGETED_DEVICE_FAMILY = "1,2,3";
+				TVOS_DEPLOYMENT_TARGET = 15.6;
 			};
 			name = Release;
 		};
@@ -404,7 +408,9 @@
 				SWIFT_OBJC_BRIDGING_HEADER = "AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h";
 				SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
 				SWIFT_VERSION = 5.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator appletvos appletvsimulator";
+				TARGETED_DEVICE_FAMILY = "1,2,3";
+				TVOS_DEPLOYMENT_TARGET = 15.6;
 				TEST_TARGET_NAME = AgentDeviceRunner;
 			};
 			name = Debug;
@@ -426,7 +432,9 @@
 				SWIFT_OBJC_BRIDGING_HEADER = "AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h";
 				SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
 				SWIFT_VERSION = 5.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
+				SUPPORTED_PLATFORMS = "iphoneos iphonesimulator appletvos appletvsimulator";
+				TARGETED_DEVICE_FAMILY = "1,2,3";
+				TVOS_DEPLOYMENT_TARGET = 15.6;
 				TEST_TARGET_NAME = AgentDeviceRunner;
 			};
 			name = Release;

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -790,7 +790,7 @@ final class RunnerTests: XCTestCase {
       performBackGesture(app: activeApp)
       return Response(ok: true, data: DataPayload(message: "back"))
     case .home:
-      XCUIDevice.shared.press(.home)
+      pressHomeButton()
       return Response(ok: true, data: DataPayload(message: "home"))
     case .appSwitcher:
       performAppSwitcherGesture(app: activeApp)
@@ -990,21 +990,43 @@ final class RunnerTests: XCTestCase {
       back.tap()
       return true
     }
+#if os(tvOS)
+    XCUIRemote.shared.press(.menu)
+    return true
+#endif
     return false
   }
 
   private func performBackGesture(app: XCUIApplication) {
+#if os(tvOS)
+    XCUIRemote.shared.press(.menu)
+#else
     let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
     let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
     let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
     start.press(forDuration: 0.05, thenDragTo: end)
+#endif
   }
 
   private func performAppSwitcherGesture(app: XCUIApplication) {
+#if os(tvOS)
+    XCUIRemote.shared.press(.home)
+    usleep(120_000)
+    XCUIRemote.shared.press(.home)
+#else
     let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
     let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
     let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
     start.press(forDuration: 0.6, thenDragTo: end)
+#endif
+  }
+
+  private func pressHomeButton() {
+#if os(tvOS)
+    XCUIRemote.shared.press(.home)
+#else
+    XCUIDevice.shared.press(.home)
+#endif
   }
 
   private func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
@@ -1109,6 +1131,19 @@ final class RunnerTests: XCTestCase {
   }
 
   private func swipe(app: XCUIApplication, direction: SwipeDirection) {
+#if os(tvOS)
+    switch direction {
+    case .up:
+      XCUIRemote.shared.press(.up)
+    case .down:
+      XCUIRemote.shared.press(.down)
+    case .left:
+      XCUIRemote.shared.press(.left)
+    case .right:
+      XCUIRemote.shared.press(.right)
+    }
+    return
+#endif
     let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
     let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2))
     let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.8))

skills/agent-device/SKILL.md

@@ -72,8 +72,8 @@ Use `--target mobile|tv` with `--platform` (required) to pick phone/tablet vs TV
 
 TV quick reference:
 - AndroidTV: `open`/`apps` use TV launcher discovery automatically.
-- tvOS: use non-runner flows only (`open`, `close`, `apps`, `screenshot`, `logs`, `reinstall`, `boot`).
-- tvOS runner-driven interactions are unsupported (`snapshot`, `wait`, `press`, `fill`, `get`, `scroll`, `back`, `home`, `app-switcher`, `record`).
+- tvOS: runner-driven interactions and snapshots are supported (`snapshot`, `wait`, `press`, `fill`, `get`, `scroll`, `back`, `home`, `app-switcher`, `record` and related selector flows).
+- tvOS `back`/`home`/`app-switcher` map to Siri Remote actions (`menu`, `home`, double-home) in the runner.
 
 ### Snapshot and targeting
 
@@ -115,7 +115,7 @@ agent-device batch --steps-file /tmp/batch-steps.json --json
 - Use `fill` for clear-then-type semantics; use `type` for focused append typing.
 - iOS `appstate` is session-scoped; Android `appstate` is live foreground state.
 - iOS settings helpers are simulator-only; use `appearance light|dark|toggle` and faceid `match|nonmatch|enroll|unenroll`.
-- For AndroidTV/tvOS selection, always pair `--target` with `--platform`; target-only selection is invalid.
+- For AndroidTV/tvOS selection, always pair `--target` with `--platform` (`ios`, `android`, or `apple` alias); target-only selection is invalid.
 - `push` simulates notification delivery:
   - iOS simulator uses APNs-style payload JSON.
   - Android uses broadcast action + typed extras (string/boolean/number).

src/core/__tests__/capabilities.test.ts

@@ -106,12 +106,12 @@ test('Android TV uses Android capabilities for core commands', () => {
   }
 });
 
-test('tvOS allows non-runner commands and blocks runner-driven interactions', () => {
+test('tvOS follows iOS capability matrix by device kind', () => {
   for (const cmd of ['open', 'close', 'apps', 'screenshot', 'logs', 'reinstall', 'boot']) {
     assert.equal(isCommandSupportedOnDevice(cmd, tvOsSimulator), true, `${cmd} on tvOS`);
   }
   for (const cmd of ['snapshot', 'wait', 'press', 'get', 'fill', 'scroll', 'back', 'home', 'app-switcher', 'record']) {
-    assert.equal(isCommandSupportedOnDevice(cmd, tvOsSimulator), false, `${cmd} on tvOS`);
+    assert.equal(isCommandSupportedOnDevice(cmd, tvOsSimulator), true, `${cmd} on tvOS`);
   }
 });
 

src/core/capabilities.ts

@@ -46,27 +46,13 @@ const COMMAND_CAPABILITY_MATRIX: Record<string, CommandCapability> = {
   wait: { ios: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true } },
 };
 
-const IOS_TV_SUPPORTED_COMMANDS = new Set<string>([
-  'apps',
-  'boot',
-  'close',
-  'logs',
-  'open',
-  'reinstall',
-  'screenshot',
-]);
-
 export function isCommandSupportedOnDevice(command: string, device: DeviceInfo): boolean {
   const capability = COMMAND_CAPABILITY_MATRIX[command];
   if (!capability) return true;
   const byPlatform = capability[device.platform];
   if (!byPlatform) return false;
   const kind = (device.kind ?? 'unknown') as keyof KindMatrix;
-  if (byPlatform[kind] !== true) return false;
-  if (device.platform === 'ios' && device.target === 'tv') {
-    return IOS_TV_SUPPORTED_COMMANDS.has(command);
-  }
-  return true;
+  return byPlatform[kind] === true;
 }
 
 export function listCapabilityCommands(): string[] {

src/core/dispatch.ts

@@ -36,8 +36,9 @@ export async function resolveTargetDevice(flags: CommandFlags): Promise<DeviceIn
   return await withDiagnosticTimer(
     'resolve_target_device',
     async () => {
+      const normalizedPlatform = flags.platform === 'apple' ? 'ios' : flags.platform;
       const selector = {
-        platform: flags.platform,
+        platform: normalizedPlatform,
         target: flags.target,
         deviceName: flags.device,
         udid: flags.udid,
@@ -46,7 +47,7 @@ export async function resolveTargetDevice(flags: CommandFlags): Promise<DeviceIn
       if (selector.target && !selector.platform) {
         throw new AppError(
           'INVALID_ARGS',
-          'Device target selector requires --platform. Use --platform ios|android with --target mobile|tv.',
+          'Device target selector requires --platform. Use --platform ios|android|apple with --target mobile|tv.',
         );
       }
 
@@ -75,7 +76,7 @@ export async function resolveTargetDevice(flags: CommandFlags): Promise<DeviceIn
       return await selectDevice(devices, selector);
     },
     {
-      platform: flags.platform,
+      platform: flags.platform === 'apple' ? 'ios' : flags.platform,
       target: flags.target,
     },
   );

src/daemon/__tests__/session-selector.test.ts

@@ -43,6 +43,20 @@ test('rejects mismatched platform selector', () => {
   );
 });
 
+test('accepts --platform apple alias for ios sessions', () => {
+  const session = makeSession({
+    device: {
+      platform: 'ios',
+      id: 'tv-sim-1',
+      name: 'Apple TV',
+      kind: 'simulator',
+      target: 'tv',
+      booted: true,
+    },
+  });
+  assert.doesNotThrow(() => assertSessionSelectorMatches(session, { platform: 'apple', target: 'tv' }));
+});
+
 test('rejects mismatched serial selector', () => {
   const session = makeSession();
   assert.throws(

src/daemon/handlers/session.ts

@@ -54,6 +54,11 @@ const REPLAY_PARENT_FLAG_KEYS: Array<keyof CommandFlags> = ['platform', 'target'
 const LOG_ACTIONS = ['path', 'start', 'stop', 'doctor', 'mark', 'clear'] as const;
 const LOG_ACTIONS_MESSAGE = `logs requires ${LOG_ACTIONS.slice(0, -1).join(', ')}, or ${LOG_ACTIONS.at(-1)}`;
 
+function normalizePlatformSelector(platform: CommandFlags['platform'] | undefined): DeviceInfo['platform'] | undefined {
+  if (platform === 'apple') return 'ios';
+  return platform;
+}
+
 function requireSessionOrExplicitSelector(
   command: string,
   session: SessionState | undefined,
@@ -85,7 +90,8 @@ function selectorTargetsSessionDevice(
 ): boolean {
   if (!session) return false;
   if (!hasExplicitDeviceSelector(flags)) return true;
-  if (flags?.platform && flags.platform !== session.device.platform) return false;
+  const normalizedPlatform = normalizePlatformSelector(flags?.platform);
+  if (normalizedPlatform && normalizedPlatform !== session.device.platform) return false;
   if (flags?.target && flags.target !== (session.device.target ?? 'mobile')) return false;
   if (flags?.udid && flags.udid !== session.device.id) return false;
   if (flags?.serial && flags.serial !== session.device.id) return false;
@@ -179,9 +185,10 @@ async function handleAppStateCommand(params: {
   const { req, sessionName, sessionStore, ensureReady, resolveDevice } = params;
   const session = sessionStore.get(sessionName);
   const flags = req.flags ?? {};
+  const normalizedPlatform = normalizePlatformSelector(flags.platform);
   if (!session && hasExplicitSessionFlag(flags)) {
     const iOSSessionHint =
-      flags.platform === 'ios'
+      normalizedPlatform === 'ios'
         ? `No active session "${sessionName}". Run open with --session ${sessionName} first.`
         : `No active session "${sessionName}". Run open with --session ${sessionName} first, or omit --session to query by device selector.`;
     return {
@@ -196,7 +203,7 @@ async function handleAppStateCommand(params: {
   if (guard) return guard;
 
   const shouldUseSessionStateForIos = session?.device.platform === 'ios' && selectorTargetsSessionDevice(flags, session);
-  const targetsIos = flags.platform === 'ios';
+  const targetsIos = normalizedPlatform === 'ios';
   if (targetsIos && !shouldUseSessionStateForIos) {
     return {
       ok: false,
@@ -315,10 +322,11 @@ export async function handleSessionCommands(params: {
   if (command === 'devices') {
     try {
       const devices: DeviceInfo[] = [];
-      if (req.flags?.platform === 'android') {
+      const requestedPlatform = normalizePlatformSelector(req.flags?.platform);
+      if (requestedPlatform === 'android') {
         const { listAndroidDevices } = await import('../../platforms/android/devices.ts');
         devices.push(...(await listAndroidDevices()));
-      } else if (req.flags?.platform === 'ios') {
+      } else if (requestedPlatform === 'ios') {
         const { listIosDevices } = await import('../../platforms/ios/devices.ts');
         devices.push(...(await listIosDevices()));
       } else {

src/daemon/session-selector.ts

@@ -2,6 +2,11 @@ import { AppError } from '../utils/errors.ts';
 import type { CommandFlags } from '../core/dispatch.ts';
 import type { SessionState } from './types.ts';
 
+function normalizePlatformSelector(platform: CommandFlags['platform'] | undefined): SessionState['device']['platform'] | undefined {
+  if (platform === 'apple') return 'ios';
+  return platform;
+}
+
 export function assertSessionSelectorMatches(
   session: SessionState,
   flags?: CommandFlags,
@@ -11,7 +16,8 @@ export function assertSessionSelectorMatches(
   const mismatches: string[] = [];
   const device = session.device;
 
-  if (flags.platform && flags.platform !== device.platform) {
+  const normalizedPlatform = normalizePlatformSelector(flags.platform);
+  if (normalizedPlatform && normalizedPlatform !== device.platform) {
     mismatches.push(`--platform=${flags.platform}`);
   }
   if (flags.target && flags.target !== (device.target ?? 'mobile')) {