fix(webview): support input for elements inside cross-origin <iframe>

Author: dcrousso

packages/injected/src/webview/webViewInput.ts

@@ -66,6 +66,10 @@ function markAndDispatch(node: EventTarget, event: Event): boolean {
   return node.dispatchEvent(event);
 }
 
+function isFrameOwner(element: Element | null): element is HTMLIFrameElement | HTMLFrameElement {
+  return !!element && (element.localName === 'iframe' || element.localName === 'frame');
+}
+
 // Legacy WebKit-only KeyboardEvent.keyIdentifier (a DOM Level 3 draft property
 // dropped by every other engine). It cannot be supplied via the constructor, so
 // compute it from the virtual key code and define it on the event before
@@ -145,6 +149,19 @@ export class WebViewInput {
     return el;
   }
 
+  positionInIFrame(x: number, y: number): { iframe: HTMLIFrameElement | HTMLFrameElement | null, x: number, y: number } {
+    const target = this._deepElementFromPoint(x, y);
+    if (!isFrameOwner(target))
+      return { iframe: null, x, y };
+    const frameRect = target.getBoundingClientRect();
+    const frameStyle = this._window.getComputedStyle(target);
+    return {
+      iframe: target,
+      x: x - frameRect.left - parseFloat(frameStyle.borderLeftWidth) - parseFloat(frameStyle.paddingLeft),
+      y: y - frameRect.top - parseFloat(frameStyle.borderTopWidth) - parseFloat(frameStyle.paddingTop),
+    };
+  }
+
   // The focused element may live inside one or more shadow roots, where
   // document.activeElement only reports the outermost shadow host.
   private _deepActiveElement(): Element | null {
@@ -154,6 +171,11 @@ export class WebViewInput {
     return active;
   }
 
+  activeIFrame(): HTMLIFrameElement | HTMLFrameElement | null {
+    const active = this._deepActiveElement();
+    return isFrameOwner(active) ? active : null;
+  }
+
   private _insertText(target: Element | null, text: string) {
     if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) {
       const start = target.selectionStart ?? target.value.length;
@@ -350,7 +372,7 @@ export class WebViewInput {
       metaKey: params.metaKey,
     };
     try {
-      const touch = new Touch({ identifier: 0, target, clientX: params.x, clientY: params.y, screenX: params.x, screenY: params.y, pageX: params.x, pageY: params.y, radiusX: 1, radiusY: 1, rotationAngle: 0, force: 1 });
+      const touch = new Touch({ identifier: 0, target, clientX: params.x, clientY: params.y, screenX: params.x, screenY: params.y, pageX: params.x + this._window.scrollX, pageY: params.y + this._window.scrollY, radiusX: 1, radiusY: 1, rotationAngle: 0, force: 1 });
       void this._postTask(() => markAndDispatch(target, new TouchEvent('touchstart', { ...init, touches: [touch], targetTouches: [touch], changedTouches: [touch] })));
       void this._postTask(() => markAndDispatch(target, new TouchEvent('touchend', { ...init, touches: [], targetTouches: [], changedTouches: [touch] })));
     } catch {

packages/playwright-core/src/server/webkit/webview/wvInput.ts

@@ -18,8 +18,10 @@
 import * as input from '../../input';
 
 import type * as types from '../../types';
-import type { WVSession } from './wvConnection';
+import type * as frames from '../../frames';
+import type { Func1 } from '../../javascript';
 import type { Progress } from '../../progress';
+import type { WVPage } from './wvPage';
 
 function modifierFlags(modifiers: Set<types.KeyboardModifier>) {
   return {
@@ -51,11 +53,16 @@ function toButtonsMask(buttons: Set<types.MouseButton>): number {
   return mask;
 }
 
+async function evaluateInFrame<Arg>(progress: Progress, frame: frames.Frame, pageFunction: Func1<Arg, void>, arg: Arg): Promise<void> {
+  const context = await progress.race(frame.mainContext());
+  await progress.race(context.evaluate(pageFunction, arg));
+}
+
 export class RawKeyboardImpl implements input.RawKeyboard {
-  private _session: WVSession | undefined;
+  private _page: WVPage;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(page: WVPage) {
+    this._page = page;
   }
 
   async keydown(progress: Progress, modifiers: Set<types.KeyboardModifier>, keyName: string, description: input.KeyDescription, autoRepeat: boolean): Promise<void> {
@@ -65,31 +72,33 @@ export class RawKeyboardImpl implements input.RawKeyboard {
       ...modifierFlags(modifiers),
       ...(text ? { text } : {}),
     };
-    await callWebViewInput(progress, this._session, 'keydown', params);
+    const frame = await this._page.deepestFocusedFrame(progress);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.keydown(p), params);
   }
 
   async keyup(progress: Progress, modifiers: Set<types.KeyboardModifier>, keyName: string, description: input.KeyDescription): Promise<void> {
     const { code, keyCode, key, location } = description;
     const params = { code, key, keyCode, location, ...modifierFlags(modifiers) };
-    await callWebViewInput(progress, this._session, 'keyup', params);
+    const frame = await this._page.deepestFocusedFrame(progress);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.keyup(p), params);
   }
 
   async sendText(progress: Progress, text: string): Promise<void> {
-    await callWebViewInput(progress, this._session, 'insertText', text);
+    const frame = await this._page.deepestFocusedFrame(progress);
+    await evaluateInFrame(progress, frame, t => (globalThis as any).__pwWebViewInput.insertText(t), text);
   }
 }
 
 export class RawMouseImpl implements input.RawMouse {
-  private _session: WVSession | undefined;
+  private _page: WVPage;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(page: WVPage) {
+    this._page = page;
   }
 
   async move(progress: Progress, x: number, y: number, button: types.MouseButton | 'none', buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, forClick: boolean): Promise<void> {
-    await callWebViewInput(progress, this._session, 'mouseMove', {
-      x, y, button: buttonToNumber(button), buttons: toButtonsMask(buttons), ...modifierFlags(modifiers),
-    });
+    const { frame, point } = await this._page.deepestFrameForPoint(progress, x, y);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.mouseMove(p), { ...point, button: buttonToNumber(button), buttons: toButtonsMask(buttons), ...modifierFlags(modifiers) });
   }
 
   async down(progress: Progress, x: number, y: number, button: types.MouseButton, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, clickCount: number): Promise<void> {
@@ -114,43 +123,27 @@ export class RawMouseImpl implements input.RawMouse {
   }
 
   async wheel(progress: Progress, x: number, y: number, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, deltaX: number, deltaY: number): Promise<void> {
-    await callWebViewInput(progress, this._session, 'wheel', { x, y, deltaX, deltaY, ...modifierFlags(modifiers) });
+    const { frame, point } = await this._page.deepestFrameForPoint(progress, x, y);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.wheel(p), { ...point, deltaX, deltaY, ...modifierFlags(modifiers) });
   }
 
   private async _mouseEvent(progress: Progress, type: string, x: number, y: number, button: number, buttons: number, modifiers: Set<types.KeyboardModifier>, clickCount: number) {
-    await callWebViewInput(progress, this._session, 'mouseEvent', {
-      type, x, y, button, buttons, clickCount, ...modifierFlags(modifiers),
+    const { frame, point } = await this._page.deepestFrameForPoint(progress, x, y);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.mouseEvent(p), {
+      type, ...point, button, buttons, clickCount, ...modifierFlags(modifiers),
     });
   }
 }
 
 export class RawTouchscreenImpl implements input.RawTouchscreen {
-  private _session: WVSession | undefined;
+  private _page: WVPage;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(page: WVPage) {
+    this._page = page;
   }
 
   async tap(progress: Progress, x: number, y: number, modifiers: Set<types.KeyboardModifier>) {
-    await callWebViewInput(progress, this._session, 'tap', { x, y, ...modifierFlags(modifiers) });
-  }
-}
-
-async function callWebViewInput(progress: Progress, session: WVSession | undefined, method: string, arg: any): Promise<void> {
-  if (!session)
-    throw new Error('Page is not initialized');
-  const expression = `window.__pwWebViewInput.${method}(${JSON.stringify(arg)})`;
-  // Some dispatchers are async — they spread events across event-loop tasks the
-  // way a real device does. Await the returned promise so the action only
-  // resolves once every event has been delivered. Stock WebKit's Runtime.evaluate
-  // has no awaitPromise option, so use the separate Runtime.awaitPromise command.
-  const { result } = await progress.race(session.send('Runtime.evaluate', { expression, returnByValue: false }));
-  // `result` is absent if evaluation failed (e.g. the frame navigated away).
-  // Only promises carry an objectId here — every __pwWebViewInput method returns
-  // void or a Promise — so this both awaits async dispatch and avoids leaking a
-  // handle for the synchronous (void) case.
-  if (result?.className === 'Promise' && result.objectId) {
-    await progress.race(session.send('Runtime.awaitPromise', { promiseObjectId: result.objectId, returnByValue: true }));
-    session.sendMayFail('Runtime.releaseObject', { objectId: result.objectId });
+    const { frame, point } = await this._page.deepestFrameForPoint(progress, x, y);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.tap(p), { ...point, ...modifierFlags(modifiers) });
   }
 }

packages/playwright-core/src/server/webkit/webview/wvPage.ts

@@ -79,9 +79,9 @@ export class WVPage implements PageDelegate {
   constructor(browserContext: WVBrowserContext, outerSession: WVSession, dialogEndpoint?: string) {
     this._outerSession = outerSession;
     this._dialogEndpoint = dialogEndpoint;
-    this.rawKeyboard = new RawKeyboardImpl();
-    this.rawMouse = new RawMouseImpl();
-    this.rawTouchscreen = new RawTouchscreenImpl();
+    this.rawKeyboard = new RawKeyboardImpl(this);
+    this.rawMouse = new RawMouseImpl(this);
+    this.rawTouchscreen = new RawTouchscreenImpl(this);
     this._contextIdToContext = new Map();
     this._page = new Page(this, browserContext);
     this._workers = new WVWorkers(this._page, outerSession);
@@ -192,9 +192,6 @@ export class WVPage implements PageDelegate {
   private _setSession(session: WVSession) {
     eventsHelper.removeEventListeners(this._sessionListeners);
     this._session = session;
-    this.rawKeyboard.setSession(session);
-    this.rawMouse.setSession(session);
-    this.rawTouchscreen.setSession(session);
     this._workers.setSession(session);
     this._addSessionListeners();
   }
@@ -917,6 +914,56 @@ export class WVPage implements PageDelegate {
   async inputActionEpilogue(): Promise<void> {
   }
 
+  async deepestFrameForPoint(progress: Progress, x: number, y: number): Promise<{ frame: frames.Frame, point: types.Point }> {
+    const path = await this.framePointerPath(progress, x, y);
+    return path[path.length - 1];
+  }
+
+  async deepestFocusedFrame(progress: Progress): Promise<frames.Frame> {
+    let frame: frames.Frame = this._page.mainFrame();
+    for (;;) {
+      const context = await progress.race(frame.mainContext());
+      const iframe = await progress.race(context.evaluateHandle(() => (globalThis as any).__pwWebViewInput.activeIFrame())) as dom.ElementHandle;
+      const childFrame = await this._childFrameAndDispose(progress, iframe);
+      if (!childFrame)
+        break;
+      frame = childFrame;
+    }
+    return frame;
+  }
+
+  // Walk from the main frame into the deepest <iframe> that contains the point,
+  // recording each frame and the point translated into that frame's coordinates.
+  async framePointerPath(progress: Progress, x: number, y: number): Promise<{ frame: frames.Frame, point: types.Point }[]> {
+    const path: { frame: frames.Frame, point: types.Point }[] = [];
+    let frame: frames.Frame = this._page.mainFrame();
+    let point: types.Point = { x, y };
+    for (;;) {
+      path.push({ frame, point });
+      const context = await progress.race(frame.mainContext());
+      const position = await progress.race(context.evaluateHandle(p => (globalThis as any).__pwWebViewInput.positionInIFrame(p.x, p.y), point));
+      try {
+        const iframe = await position.getProperty(progress, 'iframe') as dom.ElementHandle;
+        const childFrame = await this._childFrameAndDispose(progress, iframe);
+        if (!childFrame)
+          break;
+        frame = childFrame;
+        point = await progress.race(position.evaluate(result => ({ x: result.x, y: result.y })));
+      } finally {
+        position.dispose();
+      }
+    }
+    return path;
+  }
+
+  private async _childFrameAndDispose(progress: Progress, iframe: dom.ElementHandle): Promise<frames.Frame | null> {
+    try {
+      return await progress.race(this.getContentFrame(iframe));
+    } finally {
+      iframe.dispose();
+    }
+  }
+
   async resetForReuse(progress: Progress): Promise<void> {
   }