fix(webview): support input for elements inside cross-origin <iframe> (#41463)

Author: dcrousso

packages/injected/src/webview/webViewInput.ts

@@ -66,6 +66,10 @@ function markAndDispatch(node: EventTarget, event: Event): boolean {
   return node.dispatchEvent(event);
 }
 
+function isFrameOwner(element: Element | null): element is HTMLIFrameElement | HTMLFrameElement {
+  return !!element && (element.localName === 'iframe' || element.localName === 'frame');
+}
+
 // Legacy WebKit-only KeyboardEvent.keyIdentifier (a DOM Level 3 draft property
 // dropped by every other engine). It cannot be supplied via the constructor, so
 // compute it from the virtual key code and define it on the event before
@@ -145,6 +149,19 @@ export class WebViewInput {
     return el;
   }
 
+  positionInIFrame(x: number, y: number): { iframe: HTMLIFrameElement | HTMLFrameElement | null, x: number, y: number } {
+    const target = this._deepElementFromPoint(x, y);
+    if (!isFrameOwner(target))
+      return { iframe: null, x, y };
+    const frameRect = target.getBoundingClientRect();
+    const frameStyle = this._window.getComputedStyle(target);
+    return {
+      iframe: target,
+      x: x - frameRect.left - parseFloat(frameStyle.borderLeftWidth) - parseFloat(frameStyle.paddingLeft),
+      y: y - frameRect.top - parseFloat(frameStyle.borderTopWidth) - parseFloat(frameStyle.paddingTop),
+    };
+  }
+
   // The focused element may live inside one or more shadow roots, where
   // document.activeElement only reports the outermost shadow host.
   private _deepActiveElement(): Element | null {
@@ -154,6 +171,11 @@ export class WebViewInput {
     return active;
   }
 
+  activeIFrame(): HTMLIFrameElement | HTMLFrameElement | null {
+    const active = this._deepActiveElement();
+    return isFrameOwner(active) ? active : null;
+  }
+
   private _insertText(target: Element | null, text: string) {
     if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) {
       const start = target.selectionStart ?? target.value.length;
@@ -269,6 +291,7 @@ export class WebViewInput {
       metaKey: params.metaKey,
     };
     const pointer: PointerEventInit = { ...base, pointerId: 1, pointerType: 'mouse', isPrimary: true };
+    let lastTask = Promise.resolve();
     const prev = this._hoverTarget;
     if (prev !== target) {
       if (prev && prev.isConnected) {
@@ -280,13 +303,29 @@ export class WebViewInput {
       void this._postTask(() => markAndDispatch(target, new PointerEvent('pointerover', { ...pointer, relatedTarget: prev })));
       void this._postTask(() => markAndDispatch(target, new MouseEvent('mouseover', { ...base, relatedTarget: prev })));
       void this._postTask(() => markAndDispatch(target, new PointerEvent('pointerenter', { ...pointer, bubbles: false, cancelable: false, relatedTarget: prev })));
-      void this._postTask(() => markAndDispatch(target, new MouseEvent('mouseenter', { ...base, bubbles: false, cancelable: false, relatedTarget: prev })));
+      lastTask = this._postTask(() => markAndDispatch(target, new MouseEvent('mouseenter', { ...base, bubbles: false, cancelable: false, relatedTarget: prev })));
       this._hoverTarget = target;
     }
+    // Movements within an <iframe> are not dispatched in the owner context.
+    if (isFrameOwner(target))
+      return lastTask;
     void this._postTask(() => markAndDispatch(target, new PointerEvent('pointermove', pointer)));
     return this._postTask(() => markAndDispatch(target, new MouseEvent('mousemove', base)));
   }
 
+  clearHover(): Promise<void> {
+    const prev = this._hoverTarget;
+    this._hoverTarget = null;
+    if (!prev?.isConnected)
+      return Promise.resolve();
+    const base: MouseEventInit = { bubbles: true, cancelable: true, view: this._window, relatedTarget: null };
+    const pointer: PointerEventInit = { ...base, pointerId: 1, pointerType: 'mouse', isPrimary: true };
+    void this._postTask(() => markAndDispatch(prev, new PointerEvent('pointerout', pointer)));
+    void this._postTask(() => markAndDispatch(prev, new MouseEvent('mouseout', base)));
+    void this._postTask(() => markAndDispatch(prev, new PointerEvent('pointerleave', { ...pointer, bubbles: false, cancelable: false })));
+    return this._postTask(() => markAndDispatch(prev, new MouseEvent('mouseleave', { ...base, bubbles: false, cancelable: false })));
+  }
+
   mouseEvent(params: MouseEventParams): Promise<void> {
     // Resolve the hit target at dispatch time, not enqueue time: a queued move
     // ahead of this may reveal an overlay that should receive the press.
@@ -350,7 +389,7 @@ export class WebViewInput {
       metaKey: params.metaKey,
     };
     try {
-      const touch = new Touch({ identifier: 0, target, clientX: params.x, clientY: params.y, screenX: params.x, screenY: params.y, pageX: params.x, pageY: params.y, radiusX: 1, radiusY: 1, rotationAngle: 0, force: 1 });
+      const touch = new Touch({ identifier: 0, target, clientX: params.x, clientY: params.y, screenX: params.x, screenY: params.y, pageX: params.x + this._window.scrollX, pageY: params.y + this._window.scrollY, radiusX: 1, radiusY: 1, rotationAngle: 0, force: 1 });
       void this._postTask(() => markAndDispatch(target, new TouchEvent('touchstart', { ...init, touches: [touch], targetTouches: [touch], changedTouches: [touch] })));
       void this._postTask(() => markAndDispatch(target, new TouchEvent('touchend', { ...init, touches: [], targetTouches: [], changedTouches: [touch] })));
     } catch {

packages/playwright-core/src/server/webkit/webview/wvInput.ts

@@ -18,8 +18,10 @@
 import * as input from '../../input';
 
 import type * as types from '../../types';
-import type { WVSession } from './wvConnection';
+import type * as frames from '../../frames';
+import type { Func1 } from '../../javascript';
 import type { Progress } from '../../progress';
+import type { WVPage } from './wvPage';
 
 function modifierFlags(modifiers: Set<types.KeyboardModifier>) {
   return {
@@ -51,11 +53,16 @@ function toButtonsMask(buttons: Set<types.MouseButton>): number {
   return mask;
 }
 
+async function evaluateInFrame<Arg>(progress: Progress, frame: frames.Frame, pageFunction: Func1<Arg, void>, arg: Arg): Promise<void> {
+  const context = await progress.race(frame.mainContext());
+  await progress.race(context.evaluate(pageFunction, arg));
+}
+
 export class RawKeyboardImpl implements input.RawKeyboard {
-  private _session: WVSession | undefined;
+  private _page: WVPage;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(page: WVPage) {
+    this._page = page;
   }
 
   async keydown(progress: Progress, modifiers: Set<types.KeyboardModifier>, keyName: string, description: input.KeyDescription, autoRepeat: boolean): Promise<void> {
@@ -65,31 +72,48 @@ export class RawKeyboardImpl implements input.RawKeyboard {
       ...modifierFlags(modifiers),
       ...(text ? { text } : {}),
     };
-    await callWebViewInput(progress, this._session, 'keydown', params);
+    const frame = await this._page.deepestFocusedFrame(progress);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.keydown(p), params);
   }
 
   async keyup(progress: Progress, modifiers: Set<types.KeyboardModifier>, keyName: string, description: input.KeyDescription): Promise<void> {
     const { code, keyCode, key, location } = description;
     const params = { code, key, keyCode, location, ...modifierFlags(modifiers) };
-    await callWebViewInput(progress, this._session, 'keyup', params);
+    const frame = await this._page.deepestFocusedFrame(progress);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.keyup(p), params);
   }
 
   async sendText(progress: Progress, text: string): Promise<void> {
-    await callWebViewInput(progress, this._session, 'insertText', text);
+    const frame = await this._page.deepestFocusedFrame(progress);
+    await evaluateInFrame(progress, frame, t => (globalThis as any).__pwWebViewInput.insertText(t), text);
   }
 }
 
 export class RawMouseImpl implements input.RawMouse {
-  private _session: WVSession | undefined;
+  private _page: WVPage;
+  private _lastHoveredFrames: frames.Frame[] = [];
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(page: WVPage) {
+    this._page = page;
   }
 
   async move(progress: Progress, x: number, y: number, button: types.MouseButton | 'none', buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, forClick: boolean): Promise<void> {
-    await callWebViewInput(progress, this._session, 'mouseMove', {
-      x, y, button: buttonToNumber(button), buttons: toButtonsMask(buttons), ...modifierFlags(modifiers),
-    });
+    const path = await this._page.framePointerPath(progress, x, y);
+    const params = { button: buttonToNumber(button), buttons: toButtonsMask(buttons), ...modifierFlags(modifiers) };
+    // Each frame tracks its own hover target, so as the pointer crosses an <iframe>
+    // boundary it must leave the frames it is no longer within (deepest first) before
+    // entering the frames along the new path. A move that stays within the same frames
+    // leaves none of them, so no cross-frame mouseout/mouseleave is dispatched.
+    const hoveredFrames = path.map(entry => entry.frame);
+    const attachedFrames = this._page._page.frameManager.frames();
+    for (const frame of this._lastHoveredFrames.reverse()) {
+      if (hoveredFrames.includes(frame) || !attachedFrames.includes(frame))
+        continue;
+      await evaluateInFrame(progress, frame, () => (globalThis as any).__pwWebViewInput.clearHover(), undefined);
+    }
+    this._lastHoveredFrames = hoveredFrames;
+    for (const { frame, point } of path)
+      await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.mouseMove(p), { ...params, ...point });
   }
 
   async down(progress: Progress, x: number, y: number, button: types.MouseButton, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, clickCount: number): Promise<void> {
@@ -114,43 +138,27 @@ export class RawMouseImpl implements input.RawMouse {
   }
 
   async wheel(progress: Progress, x: number, y: number, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, deltaX: number, deltaY: number): Promise<void> {
-    await callWebViewInput(progress, this._session, 'wheel', { x, y, deltaX, deltaY, ...modifierFlags(modifiers) });
+    const { frame, point } = await this._page.deepestFrameForPoint(progress, x, y);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.wheel(p), { ...point, deltaX, deltaY, ...modifierFlags(modifiers) });
   }
 
   private async _mouseEvent(progress: Progress, type: string, x: number, y: number, button: number, buttons: number, modifiers: Set<types.KeyboardModifier>, clickCount: number) {
-    await callWebViewInput(progress, this._session, 'mouseEvent', {
-      type, x, y, button, buttons, clickCount, ...modifierFlags(modifiers),
+    const { frame, point } = await this._page.deepestFrameForPoint(progress, x, y);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.mouseEvent(p), {
+      type, ...point, button, buttons, clickCount, ...modifierFlags(modifiers),
     });
   }
 }
 
 export class RawTouchscreenImpl implements input.RawTouchscreen {
-  private _session: WVSession | undefined;
+  private _page: WVPage;
 
-  setSession(session: WVSession) {
-    this._session = session;
+  constructor(page: WVPage) {
+    this._page = page;
   }
 
   async tap(progress: Progress, x: number, y: number, modifiers: Set<types.KeyboardModifier>) {
-    await callWebViewInput(progress, this._session, 'tap', { x, y, ...modifierFlags(modifiers) });
-  }
-}
-
-async function callWebViewInput(progress: Progress, session: WVSession | undefined, method: string, arg: any): Promise<void> {
-  if (!session)
-    throw new Error('Page is not initialized');
-  const expression = `window.__pwWebViewInput.${method}(${JSON.stringify(arg)})`;
-  // Some dispatchers are async — they spread events across event-loop tasks the
-  // way a real device does. Await the returned promise so the action only
-  // resolves once every event has been delivered. Stock WebKit's Runtime.evaluate
-  // has no awaitPromise option, so use the separate Runtime.awaitPromise command.
-  const { result } = await progress.race(session.send('Runtime.evaluate', { expression, returnByValue: false }));
-  // `result` is absent if evaluation failed (e.g. the frame navigated away).
-  // Only promises carry an objectId here — every __pwWebViewInput method returns
-  // void or a Promise — so this both awaits async dispatch and avoids leaking a
-  // handle for the synchronous (void) case.
-  if (result?.className === 'Promise' && result.objectId) {
-    await progress.race(session.send('Runtime.awaitPromise', { promiseObjectId: result.objectId, returnByValue: true }));
-    session.sendMayFail('Runtime.releaseObject', { objectId: result.objectId });
+    const { frame, point } = await this._page.deepestFrameForPoint(progress, x, y);
+    await evaluateInFrame(progress, frame, p => (globalThis as any).__pwWebViewInput.tap(p), { ...point, ...modifierFlags(modifiers) });
   }
 }

packages/playwright-core/src/server/webkit/webview/wvPage.ts

@@ -80,9 +80,9 @@ export class WVPage implements PageDelegate {
   constructor(browserContext: WVBrowserContext, outerSession: WVSession, dialogEndpoint?: string) {
     this._outerSession = outerSession;
     this._dialogEndpoint = dialogEndpoint;
-    this.rawKeyboard = new RawKeyboardImpl();
-    this.rawMouse = new RawMouseImpl();
-    this.rawTouchscreen = new RawTouchscreenImpl();
+    this.rawKeyboard = new RawKeyboardImpl(this);
+    this.rawMouse = new RawMouseImpl(this);
+    this.rawTouchscreen = new RawTouchscreenImpl(this);
     this._contextIdToContext = new Map();
     this._page = new Page(this, browserContext);
     this._workers = new WVWorkers(this._page, outerSession);
@@ -193,9 +193,6 @@ export class WVPage implements PageDelegate {
   private _setSession(session: WVSession) {
     eventsHelper.removeEventListeners(this._sessionListeners);
     this._session = session;
-    this.rawKeyboard.setSession(session);
-    this.rawMouse.setSession(session);
-    this.rawTouchscreen.setSession(session);
     this._workers.setSession(session);
     this._addSessionListeners();
   }
@@ -923,6 +920,56 @@ export class WVPage implements PageDelegate {
   async inputActionEpilogue(): Promise<void> {
   }
 
+  async deepestFrameForPoint(progress: Progress, x: number, y: number): Promise<{ frame: frames.Frame, point: types.Point }> {
+    const path = await this.framePointerPath(progress, x, y);
+    return path[path.length - 1];
+  }
+
+  async deepestFocusedFrame(progress: Progress): Promise<frames.Frame> {
+    let frame: frames.Frame = this._page.mainFrame();
+    for (;;) {
+      const context = await progress.race(frame.mainContext());
+      const iframe = await progress.race(context.evaluateHandle(() => (globalThis as any).__pwWebViewInput.activeIFrame())) as dom.ElementHandle;
+      const childFrame = await this._childFrameAndDispose(progress, iframe);
+      if (!childFrame)
+        break;
+      frame = childFrame;
+    }
+    return frame;
+  }
+
+  // Walk from the main frame into the deepest <iframe> that contains the point,
+  // recording each frame and the point translated into that frame's coordinates.
+  async framePointerPath(progress: Progress, x: number, y: number): Promise<{ frame: frames.Frame, point: types.Point }[]> {
+    const path: { frame: frames.Frame, point: types.Point }[] = [];
+    let frame: frames.Frame = this._page.mainFrame();
+    let point: types.Point = { x, y };
+    for (;;) {
+      path.push({ frame, point });
+      const context = await progress.race(frame.mainContext());
+      const position = await progress.race(context.evaluateHandle(p => (globalThis as any).__pwWebViewInput.positionInIFrame(p.x, p.y), point));
+      try {
+        const iframe = await position.getProperty(progress, 'iframe') as dom.ElementHandle;
+        const childFrame = await this._childFrameAndDispose(progress, iframe);
+        if (!childFrame)
+          break;
+        frame = childFrame;
+        point = await progress.race(position.evaluate(result => ({ x: result.x, y: result.y })));
+      } finally {
+        position.dispose();
+      }
+    }
+    return path;
+  }
+
+  private async _childFrameAndDispose(progress: Progress, iframe: dom.ElementHandle): Promise<frames.Frame | null> {
+    try {
+      return await progress.race(this.getContentFrame(iframe));
+    } finally {
+      iframe.dispose();
+    }
+  }
+
   async resetForReuse(progress: Progress): Promise<void> {
   }
 

tests/page/page-mouse.spec.ts

@@ -337,3 +337,49 @@ it('should dispatch mouse move after context menu was opened', async ({ page, br
     }
   }
 });
+
+it('should track hover across iframe boundaries', async ({ page, headless }) => {
+  it.skip(!headless, 'headed messes up with hover');
+
+  await page.setContent(`
+    <style>
+      body, html { margin: 0; padding: 0; }
+      #parentBox { position: absolute; left: 10px; top: 10px; width: 100px; height: 100px; }
+      iframe { position: absolute; left: 200px; top: 10px; width: 200px; height: 200px; border: none; }
+    </style>
+    <div id="parentBox"></div>
+    <iframe srcdoc="
+      <style>body, html { margin: 0; padding: 0; } #childBox { width: 180px; height: 180px; }</style>
+      <div id='childBox'></div>
+      <script>
+        const box = document.querySelector('#childBox');
+        box.addEventListener('mouseenter', () => window.top.__log.push('child:enter'));
+        box.addEventListener('mouseleave', () => window.top.__log.push('child:leave'));
+      </script>
+    "></iframe>
+    <script>
+      window.__log = [];
+      const box = document.querySelector('#parentBox');
+      box.addEventListener('mouseenter', () => window.__log.push('parent:enter'));
+      box.addEventListener('mouseleave', () => window.__log.push('parent:leave'));
+    </script>
+  `);
+  await page.waitForSelector('iframe');
+  await page.frames()[1].waitForSelector('#childBox');
+  const log = () => page.evaluate(() => window['__log']);
+  const parentBox = (await page.locator('#parentBox').boundingBox())!;
+  const iframeBox = (await page.locator('iframe').boundingBox())!;
+  const parentCenter = { x: parentBox.x + parentBox.width / 2, y: parentBox.y + parentBox.height / 2 };
+  const childCenter = { x: iframeBox.x + 90, y: iframeBox.y + 90 };
+
+  await page.mouse.move(parentCenter.x, parentCenter.y);
+  await expect.poll(log).toEqual(['parent:enter']);
+
+  // Crossing into the iframe leaves the parent element and enters the child.
+  await page.mouse.move(childCenter.x, childCenter.y);
+  await expect.poll(log).toEqual(['parent:enter', 'parent:leave', 'child:enter']);
+
+  // Crossing back out leaves the child element and re-enters the parent.
+  await page.mouse.move(parentCenter.x, parentCenter.y);
+  await expect.poll(log).toEqual(['parent:enter', 'parent:leave', 'child:enter', 'child:leave', 'parent:enter']);
+});