feat: Update aria hideoutside to work with shadow doms (#9607)

* feat: update ariaHideOutside to work with shadowdom

* Add tests from old PR and fix implementation

* fix lint

* fix typescript in node_modules

Author: snowystinger

package.json

@@ -197,6 +197,7 @@
     "regenerator-runtime": "0.13.3",
     "rehype-stringify": "^9.0.4",
     "rimraf": "^6.0.1",
+    "shadow-dom-testing-library": "^1.13.1",
     "sharp": "^0.33.5",
     "storybook": "^8.6.14",
     "storybook-dark-mode": "^4.0.2",
@@ -243,7 +244,8 @@
     "lightningcss": "1.30.1",
     "react-server-dom-parcel": "canary",
     "react-test-renderer": "19.1.0",
-    "@parcel/packager-react-static": "^2.16.3"
+    "@parcel/packager-react-static": "^2.16.3",
+    "@sinclair/typebox": "0.27.10"
   },
   "@parcel/transformer-css": {
     "cssModules": {

packages/@react-aria/overlays/package.json

@@ -32,6 +32,7 @@
     "@react-aria/ssr": "^3.9.10",
     "@react-aria/utils": "^3.33.0",
     "@react-aria/visually-hidden": "^3.8.30",
+    "@react-stately/flags": "^3.1.2",
     "@react-stately/overlays": "^3.6.22",
     "@react-types/button": "^3.15.0",
     "@react-types/overlays": "^3.9.3",

packages/@react-aria/overlays/src/ariaHideOutside.ts

@@ -10,7 +10,9 @@
  * governing permissions and limitations under the License.
  */
 
-import {getOwnerWindow, nodeContains} from '@react-aria/utils';
+import {createShadowTreeWalker, getOwnerDocument, getOwnerWindow, nodeContains} from '@react-aria/utils';
+import {shadowDOM} from '@react-stately/flags';
+
 const supportsInert = typeof HTMLElement !== 'undefined' && 'inert' in HTMLElement.prototype;
 
 interface AriaHideOutsideOptions {
@@ -64,6 +66,22 @@ export function ariaHideOutside(targets: Element[], options?: AriaHideOutsideOpt
     }
   };
 
+  let shadowRootsToWatch = new Set<ShadowRoot>();
+  if (shadowDOM()) {
+    // find all shadow roots that are ancestors of the targets
+    // traverse upwards until the root is reached
+    for (let target of targets) {
+      let node = target;
+      while (node && node !== root) {
+        let root = node.getRootNode();
+        if ('shadowRoot' in root) {
+          shadowRootsToWatch.add(root.shadowRoot as ShadowRoot);
+        }
+        node = root.parentNode as Element;
+      }
+    }
+  }
+
   let walk = (root: Element) => {
     // Keep live announcer and top layer elements (e.g. toasts) visible.
     for (let element of root.querySelectorAll('[data-live-announcer], [data-react-aria-top-layer]')) {
@@ -93,7 +111,8 @@ export function ariaHideOutside(targets: Element[], options?: AriaHideOutsideOpt
       return NodeFilter.FILTER_ACCEPT;
     };
 
-    let walker = document.createTreeWalker(
+    let walker = createShadowTreeWalker(
+      getOwnerDocument(root),
       root,
       NodeFilter.SHOW_ELEMENT,
       {acceptNode}
@@ -164,10 +183,65 @@ export function ariaHideOutside(targets: Element[], options?: AriaHideOutsideOpt
           }
         }
       }
+
+      if (shadowDOM()) {
+        // if any of the observed shadow roots were removed, stop observing them
+        for (let shadowRoot of shadowRootsToWatch) {
+          if (!shadowRoot.isConnected) {
+            observer.disconnect();
+            break;
+          }
+        }
+      }
     }
   });
 
   observer.observe(root, {childList: true, subtree: true});
+  let shadowObservers = new Set<MutationObserver>();
+  if (shadowDOM()) {
+    for (let shadowRoot of shadowRootsToWatch) {
+      // Disconnect single target instead of all https://github.com/whatwg/dom/issues/126
+      let shadowObserver = new MutationObserver(changes => {
+        for (let change of changes) {
+          if (change.type !== 'childList') {
+            continue;
+          }
+
+          // If the parent element of the added nodes is not within one of the targets,
+          // and not already inside a hidden node, hide all of the new children.
+          if (
+            change.target.isConnected &&
+            ![...visibleNodes, ...hiddenNodes].some((node) =>
+              nodeContains(node, change.target)
+            )
+          ) {
+            for (let node of change.addedNodes) {
+              if (
+                (node instanceof HTMLElement || node instanceof SVGElement) &&
+                (node.dataset.liveAnnouncer === 'true' || node.dataset.reactAriaTopLayer === 'true')
+              ) {
+                visibleNodes.add(node);
+              } else if (node instanceof Element) {
+                walk(node);
+              }
+            }
+          }
+
+          if (shadowDOM()) {
+            // if any of the observed shadow roots were removed, stop observing them
+            for (let shadowRoot of shadowRootsToWatch) {
+              if (!shadowRoot.isConnected) {
+                observer.disconnect();
+                break;
+              }
+            }
+          }
+        }
+      });
+      shadowObserver.observe(shadowRoot, {childList: true, subtree: true});
+      shadowObservers.add(shadowObserver);
+    }
+  }
 
   let observerWrapper: ObserverWrapper = {
     visibleNodes,
@@ -184,6 +258,11 @@ export function ariaHideOutside(targets: Element[], options?: AriaHideOutsideOpt
 
   return (): void => {
     observer.disconnect();
+    if (shadowDOM()) {
+      for (let shadowObserver of shadowObservers) {
+        shadowObserver.disconnect();
+      }
+    }
 
     for (let node of hiddenNodes) {
       let count = refCountMap.get(node);