fix(webview): support input for elements inside cross-origin <iframe>

Author: dcrousso

packages/injected/src/webview/webViewInput.ts

@@ -66,6 +66,10 @@ function markAndDispatch(node: EventTarget, event: Event): boolean {
   return node.dispatchEvent(event);
 }
 
+function isFrameOwner(element: Element): element is HTMLIFrameElement | HTMLFrameElement {
+  return element.localName === 'iframe' || element.localName === 'frame';
+}
+
 // Legacy WebKit-only KeyboardEvent.keyIdentifier (a DOM Level 3 draft property
 // dropped by every other engine). It cannot be supplied via the constructor, so
 // compute it from the virtual key code and define it on the event before
@@ -132,17 +136,29 @@ export class WebViewInput {
     });
   }
 
-  // Descend through open shadow roots so synthetic events land on the actual
-  // element under the pointer rather than on the shadow host.
-  private _deepElementFromPoint(x: number, y: number): Element | null {
+  private _hitTest(x: number, y: number): { target: Element | null, iframe: HTMLIFrameElement | HTMLFrameElement | null, x: number, y: number } {
     let el = this._document.elementFromPoint(x, y);
     while (el && el.shadowRoot) {
       const inner = el.shadowRoot.elementFromPoint(x, y);
       if (!inner || inner === el)
         break;
       el = inner;
     }
-    return el;
+    if (!el || !isFrameOwner(el))
+      return { target: el, iframe: null, x, y };
+    const frameRect = el.getBoundingClientRect();
+    const frameStyle = this._window.getComputedStyle(el);
+    return {
+      target: el,
+      iframe: el,
+      x: x - frameRect.left - parseFloat(frameStyle.borderLeftWidth) - parseFloat(frameStyle.paddingLeft),
+      y: y - frameRect.top - parseFloat(frameStyle.borderTopWidth) - parseFloat(frameStyle.paddingTop),
+    };
+  }
+
+  positionInIFrame(x: number, y: number): { iframe: HTMLIFrameElement | HTMLFrameElement | null, x: number, y: number } {
+    const hit = this._hitTest(x, y);
+    return { iframe: hit.iframe, x: hit.x, y: hit.y };
   }
 
   // The focused element may live inside one or more shadow roots, where
@@ -154,6 +170,11 @@ export class WebViewInput {
     return active;
   }
 
+  activeIFrame(): HTMLIFrameElement | HTMLFrameElement | null {
+    const active = this._deepActiveElement();
+    return active && isFrameOwner(active) ? active : null;
+  }
+
   private _insertText(target: Element | null, text: string) {
     if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) {
       const start = target.selectionStart ?? target.value.length;
@@ -252,7 +273,7 @@ export class WebViewInput {
   }
 
   mouseMove(params: MouseMoveParams): Promise<void> {
-    const target = this._deepElementFromPoint(params.x, params.y) || this._document.documentElement;
+    const target = this._hitTest(params.x, params.y).target || this._document.documentElement;
     const base: MouseEventInit = {
       bubbles: true,
       cancelable: true,
@@ -291,7 +312,7 @@ export class WebViewInput {
     // Resolve the hit target at dispatch time, not enqueue time: a queued move
     // ahead of this may reveal an overlay that should receive the press.
     return this._postTask(() => {
-      const target = this._deepElementFromPoint(params.x, params.y) || this._document.documentElement;
+      const target = this._hitTest(params.x, params.y).target || this._document.documentElement;
       markAndDispatch(target, new MouseEvent(params.type, {
         bubbles: true,
         cancelable: true,
@@ -313,7 +334,7 @@ export class WebViewInput {
 
   wheel(params: WheelParams): Promise<void> {
     return this._postTask(() => {
-      const target = this._deepElementFromPoint(params.x, params.y) || this._document.documentElement;
+      const target = this._hitTest(params.x, params.y).target || this._document.documentElement;
       markAndDispatch(target, new WheelEvent('wheel', {
         bubbles: true,
         cancelable: true,
@@ -335,7 +356,7 @@ export class WebViewInput {
   }
 
   tap(params: TapParams): Promise<void> {
-    const target = this._deepElementFromPoint(params.x, params.y) || this._document.documentElement;
+    const target = this._hitTest(params.x, params.y).target || this._document.documentElement;
     const init: MouseEventInit = {
       bubbles: true,
       cancelable: true,
@@ -350,7 +371,7 @@ export class WebViewInput {
       metaKey: params.metaKey,
     };
     try {
-      const touch = new Touch({ identifier: 0, target, clientX: params.x, clientY: params.y, screenX: params.x, screenY: params.y, pageX: params.x, pageY: params.y, radiusX: 1, radiusY: 1, rotationAngle: 0, force: 1 });
+      const touch = new Touch({ identifier: 0, target, clientX: params.x, clientY: params.y, screenX: params.x, screenY: params.y, pageX: params.x + this._window.scrollX, pageY: params.y + this._window.scrollY, radiusX: 1, radiusY: 1, rotationAngle: 0, force: 1 });
       void this._postTask(() => markAndDispatch(target, new TouchEvent('touchstart', { ...init, touches: [touch], targetTouches: [touch], changedTouches: [touch] })));
       void this._postTask(() => markAndDispatch(target, new TouchEvent('touchend', { ...init, touches: [], targetTouches: [], changedTouches: [touch] })));
     } catch {

packages/playwright-core/src/server/webkit/webview/wvInput.ts

@@ -18,7 +18,6 @@
 import * as input from '../../input';
 
 import type * as types from '../../types';
-import type { WVSession } from './wvConnection';
 import type { Progress } from '../../progress';
 
 function modifierFlags(modifiers: Set<types.KeyboardModifier>) {
@@ -51,11 +50,13 @@ function toButtonsMask(buttons: Set<types.MouseButton>): number {
   return mask;
 }
 
+export type DispatchWebViewInput = (progress: Progress, method: string, params: any) => Promise<void>;
+
 export class RawKeyboardImpl implements input.RawKeyboard {
-  private _session: WVSession | undefined;
+  private _dispatcher: DispatchWebViewInput;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(dispatcher: DispatchWebViewInput) {
+    this._dispatcher = dispatcher;
   }
 
   async keydown(progress: Progress, modifiers: Set<types.KeyboardModifier>, keyName: string, description: input.KeyDescription, autoRepeat: boolean): Promise<void> {
@@ -65,29 +66,29 @@ export class RawKeyboardImpl implements input.RawKeyboard {
       ...modifierFlags(modifiers),
       ...(text ? { text } : {}),
     };
-    await callWebViewInput(progress, this._session, 'keydown', params);
+    await this._dispatcher(progress, 'keydown', params);
   }
 
   async keyup(progress: Progress, modifiers: Set<types.KeyboardModifier>, keyName: string, description: input.KeyDescription): Promise<void> {
     const { code, keyCode, key, location } = description;
     const params = { code, key, keyCode, location, ...modifierFlags(modifiers) };
-    await callWebViewInput(progress, this._session, 'keyup', params);
+    await this._dispatcher(progress, 'keyup', params);
   }
 
   async sendText(progress: Progress, text: string): Promise<void> {
-    await callWebViewInput(progress, this._session, 'insertText', text);
+    await this._dispatcher(progress, 'insertText', text);
   }
 }
 
 export class RawMouseImpl implements input.RawMouse {
-  private _session: WVSession | undefined;
+  private _dispatcher: DispatchWebViewInput;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(dispatcher: DispatchWebViewInput) {
+    this._dispatcher = dispatcher;
   }
 
   async move(progress: Progress, x: number, y: number, button: types.MouseButton | 'none', buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, forClick: boolean): Promise<void> {
-    await callWebViewInput(progress, this._session, 'mouseMove', {
+    await this._dispatcher(progress, 'mouseMove', {
       x, y, button: buttonToNumber(button), buttons: toButtonsMask(buttons), ...modifierFlags(modifiers),
     });
   }
@@ -114,43 +115,24 @@ export class RawMouseImpl implements input.RawMouse {
   }
 
   async wheel(progress: Progress, x: number, y: number, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, deltaX: number, deltaY: number): Promise<void> {
-    await callWebViewInput(progress, this._session, 'wheel', { x, y, deltaX, deltaY, ...modifierFlags(modifiers) });
+    await this._dispatcher(progress, 'wheel', { x, y, deltaX, deltaY, ...modifierFlags(modifiers) });
   }
 
   private async _mouseEvent(progress: Progress, type: string, x: number, y: number, button: number, buttons: number, modifiers: Set<types.KeyboardModifier>, clickCount: number) {
-    await callWebViewInput(progress, this._session, 'mouseEvent', {
+    await this._dispatcher(progress, 'mouseEvent', {
       type, x, y, button, buttons, clickCount, ...modifierFlags(modifiers),
     });
   }
 }
 
 export class RawTouchscreenImpl implements input.RawTouchscreen {
-  private _session: WVSession | undefined;
+  private _dispatcher: DispatchWebViewInput;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(dispatcher: DispatchWebViewInput) {
+    this._dispatcher = dispatcher;
   }
 
   async tap(progress: Progress, x: number, y: number, modifiers: Set<types.KeyboardModifier>) {
-    await callWebViewInput(progress, this._session, 'tap', { x, y, ...modifierFlags(modifiers) });
-  }
-}
-
-async function callWebViewInput(progress: Progress, session: WVSession | undefined, method: string, arg: any): Promise<void> {
-  if (!session)
-    throw new Error('Page is not initialized');
-  const expression = `window.__pwWebViewInput.${method}(${JSON.stringify(arg)})`;
-  // Some dispatchers are async — they spread events across event-loop tasks the
-  // way a real device does. Await the returned promise so the action only
-  // resolves once every event has been delivered. Stock WebKit's Runtime.evaluate
-  // has no awaitPromise option, so use the separate Runtime.awaitPromise command.
-  const { result } = await progress.race(session.send('Runtime.evaluate', { expression, returnByValue: false }));
-  // `result` is absent if evaluation failed (e.g. the frame navigated away).
-  // Only promises carry an objectId here — every __pwWebViewInput method returns
-  // void or a Promise — so this both awaits async dispatch and avoids leaking a
-  // handle for the synchronous (void) case.
-  if (result?.className === 'Promise' && result.objectId) {
-    await progress.race(session.send('Runtime.awaitPromise', { promiseObjectId: result.objectId, returnByValue: true }));
-    session.sendMayFail('Runtime.releaseObject', { objectId: result.objectId });
+    await this._dispatcher(progress, 'tap', { x, y, ...modifierFlags(modifiers) });
   }
 }

packages/playwright-core/src/server/webkit/webview/wvPage.ts

@@ -79,9 +79,10 @@ export class WVPage implements PageDelegate {
   constructor(browserContext: WVBrowserContext, outerSession: WVSession, dialogEndpoint?: string) {
     this._outerSession = outerSession;
     this._dialogEndpoint = dialogEndpoint;
-    this.rawKeyboard = new RawKeyboardImpl();
-    this.rawMouse = new RawMouseImpl();
-    this.rawTouchscreen = new RawTouchscreenImpl();
+    const dispatchInput = this._dispatchWebViewInput.bind(this);
+    this.rawKeyboard = new RawKeyboardImpl(dispatchInput);
+    this.rawMouse = new RawMouseImpl(dispatchInput);
+    this.rawTouchscreen = new RawTouchscreenImpl(dispatchInput);
     this._contextIdToContext = new Map();
     this._page = new Page(this, browserContext);
     this._workers = new WVWorkers(this._page, outerSession);
@@ -192,9 +193,6 @@ export class WVPage implements PageDelegate {
   private _setSession(session: WVSession) {
     eventsHelper.removeEventListeners(this._sessionListeners);
     this._session = session;
-    this.rawKeyboard.setSession(session);
-    this.rawMouse.setSession(session);
-    this.rawTouchscreen.setSession(session);
     this._workers.setSession(session);
     this._addSessionListeners();
   }
@@ -917,6 +915,40 @@ export class WVPage implements PageDelegate {
   async inputActionEpilogue(): Promise<void> {
   }
 
+  private async _dispatchWebViewInput(progress: Progress, method: string, params: any): Promise<void> {
+    const positional = typeof params === 'object' && params !== null && 'x' in params;
+    let frame: frames.Frame = this._page.mainFrame();
+    let x = positional ? params.x as number : 0;
+    let y = positional ? params.y as number : 0;
+    for (;;) {
+      const context = await progress.race(frame.mainContext());
+      const position = positional
+        ? await progress.race(context.evaluateHandle(([px, py]) => (globalThis as any).__pwWebViewInput.positionInIFrame(px, py), [x, y]))
+        : undefined;
+      try {
+        const iframe = (position
+          ? await position.getProperty(progress, 'iframe')
+          : await progress.race(context.evaluateHandle(() => (globalThis as any).__pwWebViewInput.activeIFrame()))) as dom.ElementHandle;
+        let childFrame: frames.Frame | null;
+        try {
+          childFrame = await progress.race(this.getContentFrame(iframe));
+        } finally {
+          iframe.dispose();
+        }
+        if (!childFrame)
+          break;
+        if (position)
+          ({ x, y } = await progress.race(position.evaluate(result => ({ x: result.x, y: result.y }))));
+        frame = childFrame;
+      } finally {
+        position?.dispose();
+      }
+    }
+    const context = await progress.race(frame.mainContext());
+    // This will automatically await if the result is a promise, so async dispatchers deliver every event before this resolves.
+    await progress.race(context.evaluate(({ method, params }) => (globalThis as any).__pwWebViewInput[method](params), { method, params: positional ? { ...params, x, y } : params }));
+  }
+
   async resetForReuse(progress: Progress): Promise<void> {
   }