feat: add semantic web provider seam

Author: thymikee

src/core/__tests__/web-interactor.test.ts

@@ -0,0 +1,95 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import { createWebInteractor } from '../interactors/web.ts';
+import { AppError } from '../../utils/errors.ts';
+import { withWebProvider, type WebProvider } from '../../platforms/web/provider.ts';
+
+test('web interactor delegates first-slice operations to the scoped provider', async () => {
+  const calls: string[] = [];
+  const interactor = createWebInteractor();
+  const provider = makeWebProvider({
+    async open(target, options) {
+      calls.push(`open:${target}:${options?.url ?? ''}`);
+    },
+    async close(target) {
+      calls.push(`close:${target ?? ''}`);
+    },
+    async snapshot(options) {
+      calls.push(`snapshot:${options?.scope ?? ''}`);
+      return {
+        nodes: [{ index: 0, role: 'button', label: 'Submit' }],
+        truncated: true,
+      };
+    },
+    async screenshot(outPath, options) {
+      calls.push(`screenshot:${outPath}:${options?.fullscreen === true}`);
+    },
+    async click(x, y) {
+      calls.push(`click:${x}:${y}`);
+    },
+    async fill(x, y, text, options) {
+      calls.push(`fill:${x}:${y}:${text}:${options?.delayMs ?? 0}`);
+    },
+    async typeText(text, options) {
+      calls.push(`type:${text}:${options?.delayMs ?? 0}`);
+    },
+    async scroll(direction, options) {
+      calls.push(`scroll:${direction}:${options?.pixels ?? options?.amount ?? ''}`);
+    },
+  });
+
+  const snapshot = await withWebProvider(provider, async () => {
+    await interactor.open('https://example.test');
+    await interactor.open('app-shell', { url: 'https://example.test/deep' });
+    await interactor.close('app-shell');
+    await interactor.tap(10, 20);
+    await interactor.focus(11, 21);
+    await interactor.fill(12, 22, 'hello', 5);
+    await interactor.type('world', 6);
+    await interactor.scroll('down', { pixels: 400 });
+    await interactor.screenshot('/tmp/web.png', { fullscreen: true });
+    return await interactor.snapshot({ scope: 'main' });
+  });
+
+  assert.deepEqual(calls, [
+    'open:https://example.test:',
+    'open:https://example.test/deep:https://example.test/deep',
+    'close:app-shell',
+    'click:10:20',
+    'click:11:21',
+    'fill:12:22:hello:5',
+    'type:world:6',
+    'scroll:down:400',
+    'screenshot:/tmp/web.png:true',
+    'snapshot:main',
+  ]);
+  assert.equal(snapshot.backend, 'web');
+  assert.equal(snapshot.truncated, true);
+  assert.deepEqual(snapshot.nodes, [{ index: 0, role: 'button', label: 'Submit' }]);
+});
+
+test('web interactor reports unsupported operations explicitly', async () => {
+  const interactor = createWebInteractor();
+
+  await assert.rejects(
+    () => interactor.back(),
+    (error: unknown) =>
+      error instanceof AppError &&
+      error.code === 'UNSUPPORTED_OPERATION' &&
+      error.message === 'back is not supported on web',
+  );
+});
+
+function makeWebProvider(overrides: Partial<WebProvider> = {}): WebProvider {
+  return {
+    open: async () => {},
+    close: async () => {},
+    snapshot: async () => ({ nodes: [] }),
+    screenshot: async () => {},
+    click: async () => {},
+    fill: async () => {},
+    typeText: async () => {},
+    scroll: async () => {},
+    ...overrides,
+  };
+}

src/core/interactor-types.ts

@@ -57,7 +57,7 @@ export type SnapshotOptions = BaseSnapshotOptions & {
 
 export type SnapshotResult = Omit<BackendSnapshotResult, 'backend' | 'nodes'> & {
   nodes?: RawSnapshotNode[];
-  backend: Extract<SnapshotBackend, 'android' | 'xctest' | 'linux-atspi'>;
+  backend: Extract<SnapshotBackend, 'android' | 'xctest' | 'linux-atspi' | 'web'>;
 };
 
 export type Interactor = {

src/core/interactors.ts

@@ -15,6 +15,10 @@ export async function getInteractor(
       const { createLinuxInteractor } = await import('./interactors/linux.ts');
       return createLinuxInteractor();
     }
+    case 'web': {
+      const { createWebInteractor } = await import('./interactors/web.ts');
+      return createWebInteractor();
+    }
     case 'ios':
     case 'macos': {
       const { createAppleInteractor } = await import('./interactors/apple.ts');

src/core/interactors/web.ts

@@ -0,0 +1,50 @@
+import type { Interactor } from '../interactor-types.ts';
+import { AppError } from '../../utils/errors.ts';
+import { withDiagnosticTimer } from '../../utils/diagnostics.ts';
+import { resolveWebProvider } from '../../platforms/web/provider.ts';
+
+export function createWebInteractor(): Interactor {
+  const provider = () => resolveWebProvider();
+  return {
+    open: (target, options) => provider().open(options?.url ?? target, { url: options?.url }),
+    openDevice: () => provider().open('about:blank'),
+    close: (target) => provider().close(target),
+    tap: (x, y) => provider().click(x, y),
+    doubleTap: () => unsupportedWebOperation('doubleTap'),
+    swipe: () => unsupportedWebOperation('swipe'),
+    pan: () => unsupportedWebOperation('pan'),
+    fling: () => unsupportedWebOperation('fling'),
+    longPress: () => unsupportedWebOperation('longPress'),
+    focus: (x, y) => provider().click(x, y),
+    type: (text, delayMs) => provider().typeText(text, { delayMs }),
+    fill: (x, y, text, delayMs) => provider().fill(x, y, text, { delayMs }),
+    scroll: (direction, options) => provider().scroll(direction, options),
+    pinch: () => unsupportedWebOperation('pinch'),
+    screenshot: (outPath, options) => provider().screenshot(outPath, options),
+    snapshot: async (options) => {
+      const result = await withDiagnosticTimer(
+        'snapshot_capture',
+        async () => await provider().snapshot(options),
+        { backend: 'web' },
+      );
+      return {
+        nodes: result.nodes,
+        truncated: result.truncated ?? false,
+        backend: 'web',
+      };
+    },
+    back: () => unsupportedWebOperation('back'),
+    home: () => unsupportedWebOperation('home'),
+    rotate: () => unsupportedWebOperation('rotate'),
+    rotateGesture: () => unsupportedWebOperation('rotateGesture'),
+    transformGesture: () => unsupportedWebOperation('transformGesture'),
+    appSwitcher: () => unsupportedWebOperation('appSwitcher'),
+    readClipboard: () => unsupportedWebOperation('readClipboard'),
+    writeClipboard: () => unsupportedWebOperation('writeClipboard'),
+    setSetting: () => unsupportedWebOperation('setSetting'),
+  };
+}
+
+async function unsupportedWebOperation(operation: string): Promise<never> {
+  throw new AppError('UNSUPPORTED_OPERATION', `${operation} is not supported on web`);
+}

src/daemon/__tests__/request-platform-providers.test.ts

@@ -3,11 +3,14 @@ import { test } from 'vitest';
 import {
   ANDROID_EMULATOR,
   IOS_SIMULATOR,
+  WEB_DESKTOP_DEVICE,
   makeAndroidSession,
   makeIosSession,
+  makeSession,
 } from '../../__tests__/test-utils/index.ts';
 import { withTargetDeviceResolutionScope } from '../../core/dispatch-resolve.ts';
 import { createLocalAppleToolProvider, runXcrun } from '../../platforms/ios/tool-provider.ts';
+import { resolveWebProvider, type WebProvider } from '../../platforms/web/provider.ts';
 import type { DeviceInfo } from '../../utils/device.ts';
 import { withRequestPlatformProviderScope } from '../request-platform-providers.ts';
 import type { DaemonRequest } from '../types.ts';
@@ -194,6 +197,66 @@ test('request platform provider scopes stay isolated across concurrent requests'
   assert.deepEqual(appleCalls, [`ios-session:${IOS_SIMULATOR.id}:list devices -j`]);
 });
 
+test('request platform provider scope applies web provider only for web sessions', async () => {
+  const calls: string[] = [];
+  const webProvider = makeWebProvider({
+    async open(target) {
+      calls.push(`open:${target}`);
+    },
+  });
+
+  await withRequestPlatformProviderScope(
+    {
+      req: request('open'),
+      existingSession: makeSession('web-session', { device: WEB_DESKTOP_DEVICE }),
+      providers: {
+        webProvider: ({ device, session }) => {
+          calls.push(`${session?.name}:${device.id}`);
+          return webProvider;
+        },
+        linuxToolProvider: () => {
+          throw new Error('Linux provider should not apply to a web session');
+        },
+      },
+    },
+    async () => await resolveWebProvider().open('https://example.test'),
+  );
+
+  assert.deepEqual(calls, ['web-session:agent-browser-chrome', 'open:https://example.test']);
+});
+
+test('request platform provider scope follows explicit web selector', async () => {
+  const seenDevices: string[] = [];
+
+  await withTargetDeviceResolutionScope(
+    async () => [WEB_DESKTOP_DEVICE],
+    async () =>
+      await withRequestPlatformProviderScope(
+        {
+          req: {
+            ...request('snapshot'),
+            flags: {
+              platform: 'web',
+            },
+          },
+          existingSession: undefined,
+          providers: {
+            webProvider: ({ device, session }) => {
+              seenDevices.push(`${session?.name ?? 'none'}:${device.id}`);
+              return makeWebProvider();
+            },
+            appleToolProvider: () => {
+              throw new Error('Apple provider should not apply to a web request');
+            },
+          },
+        },
+        async () => await resolveWebProvider().snapshot(),
+      ),
+  );
+
+  assert.deepEqual(seenDevices, ['none:agent-browser-chrome']);
+});
+
 function request(command: string): DaemonRequest {
   return {
     token: 'test-token',
@@ -204,3 +267,17 @@ function request(command: string): DaemonRequest {
     meta: { requestId: `req-${command}` },
   };
 }
+
+function makeWebProvider(overrides: Partial<WebProvider> = {}): WebProvider {
+  return {
+    open: async () => {},
+    close: async () => {},
+    snapshot: async () => ({ nodes: [] }),
+    screenshot: async () => {},
+    click: async () => {},
+    fill: async () => {},
+    typeText: async () => {},
+    scroll: async () => {},
+    ...overrides,
+  };
+}

src/daemon/request-platform-providers.ts

@@ -9,6 +9,7 @@ import type {
   AppleToolProvider,
 } from '../platforms/ios/tool-provider.ts';
 import type { LinuxToolProvider } from '../platforms/linux/tool-provider.ts';
+import type { WebProvider } from '../platforms/web/provider.ts';
 import { isApplePlatform, type DeviceInfo } from '../utils/device.ts';
 import type { AppLogProvider } from './app-log.ts';
 import { hasExplicitDeviceSelector } from './device-selector-intent.ts';
@@ -39,6 +40,8 @@ export type AppleToolProviderResolver = PlatformProviderResolver<
 
 export type LinuxToolProviderResolver = PlatformProviderResolver<LinuxToolProvider | undefined>;
 
+export type WebProviderResolver = PlatformProviderResolver<WebProvider | undefined>;
+
 export type AppLogProviderResolver = PlatformProviderResolver<AppLogProvider | undefined>;
 
 export type RecordingProviderResolver = PlatformProviderResolver<RecordingProvider | undefined>;
@@ -48,6 +51,7 @@ export type PlatformProviderResolvers = {
   appleRunnerProvider?: AppleRunnerProviderResolver;
   appleToolProvider?: AppleToolProviderResolver;
   linuxToolProvider?: LinuxToolProviderResolver;
+  webProvider?: WebProviderResolver;
   appLogProvider?: AppLogProviderResolver;
   recordingProvider?: RecordingProviderResolver;
 };
@@ -85,6 +89,9 @@ type ResolvedRequestPlatformProviders = {
   linuxTool?: {
     provider?: LinuxToolProvider;
   };
+  web?: {
+    provider?: WebProvider;
+  };
   appLog?: {
     provider?: AppLogProvider;
   };
@@ -184,6 +191,19 @@ const REQUEST_PLATFORM_PROVIDER_DESCRIPTORS = [
       appendRequestProviderWrapper(wrappers, scopedProviders.linuxTool, withLinuxToolProvider);
     },
   },
+  {
+    resolverKey: 'webProvider',
+    resolve(providers, context) {
+      const webProvider = providers.webProvider;
+      if (!webProvider || context.device.platform !== 'web') return {};
+      return { web: { provider: webProvider(context) } };
+    },
+    async appendWrapper(scopedProviders, wrappers) {
+      if (!scopedProviders.web?.provider) return;
+      const { withWebProvider } = await import('../platforms/web/provider.ts');
+      appendRequestProviderWrapper(wrappers, scopedProviders.web, withWebProvider);
+    },
+  },
   {
     resolverKey: 'appLogProvider',
     resolve(providers, context) {

src/platforms/web/provider.ts

@@ -0,0 +1,71 @@
+import type { ScrollDirection } from '../../core/scroll-gesture.ts';
+import type { SessionSurface } from '../../core/session-surface.ts';
+import { AppError } from '../../utils/errors.ts';
+import { createScopedProvider } from '../../utils/scoped-provider.ts';
+import type { RawSnapshotNode } from '../../utils/snapshot.ts';
+
+export type WebOpenOptions = {
+  url?: string;
+};
+
+export type WebScreenshotOptions = {
+  fullscreen?: boolean;
+  stabilize?: boolean;
+  surface?: SessionSurface;
+};
+
+export type WebSnapshotOptions = {
+  interactiveOnly?: boolean;
+  depth?: number;
+  scope?: string;
+  raw?: boolean;
+  surface?: SessionSurface;
+};
+
+export type WebSnapshotResult = {
+  nodes: RawSnapshotNode[];
+  truncated?: boolean;
+};
+
+export type WebProvider = {
+  open(target: string, options?: WebOpenOptions): Promise<void>;
+  close(target?: string): Promise<void>;
+  snapshot(options?: WebSnapshotOptions): Promise<WebSnapshotResult>;
+  screenshot(outPath: string, options?: WebScreenshotOptions): Promise<void>;
+  click(x: number, y: number): Promise<void>;
+  fill(x: number, y: number, text: string, options?: { delayMs?: number }): Promise<void>;
+  typeText(text: string, options?: { delayMs?: number }): Promise<void>;
+  scroll(direction: ScrollDirection, options?: { amount?: number; pixels?: number }): Promise<void>;
+  readText?(x: number, y: number): Promise<string>;
+};
+
+const localWebProvider: WebProvider = {
+  open: () => unsupportedLocalWebProvider(),
+  close: () => unsupportedLocalWebProvider(),
+  snapshot: () => unsupportedLocalWebProvider(),
+  screenshot: () => unsupportedLocalWebProvider(),
+  click: () => unsupportedLocalWebProvider(),
+  fill: () => unsupportedLocalWebProvider(),
+  typeText: () => unsupportedLocalWebProvider(),
+  scroll: () => unsupportedLocalWebProvider(),
+};
+
+const webProviderScope = createScopedProvider(localWebProvider);
+
+export function resolveWebProvider(provider?: WebProvider): WebProvider {
+  return webProviderScope.resolve(provider);
+}
+
+export async function withWebProvider<T>(
+  provider: WebProvider | undefined,
+  fn: () => Promise<T>,
+): Promise<T> {
+  return await webProviderScope.run(provider, fn);
+}
+
+async function unsupportedLocalWebProvider(): Promise<never> {
+  throw new AppError(
+    'UNSUPPORTED_OPERATION',
+    'Web automation requires a request-scoped web provider.',
+  );
+}