[WebGPU-XR][Phase 2] WebGPU projection layer + render target provider (#18640)

packages/dev/core/src/Engines/webgpuEngine.pure.ts

@@ -2585,6 +2585,13 @@ export class WebGPUEngine extends ThinWebGPUEngine {
      */
     public wrapWebGPUTexture(texture: GPUTexture): InternalTexture {
         const hardwareTexture = new WebGPUHardwareTexture(this, texture);
+        // Report the real format of the wrapped texture. The hardware texture otherwise keeps its
+        // default (RGBA8Unorm); the render-pass color attachment view + pipeline color target are
+        // built from hardwareTexture.format (see _setColorFormat / bindFramebuffer), so an external
+        // texture created with any other format (e.g. bgra8unorm or an *-srgb variant, as returned by
+        // XRGPUBinding.getPreferredColorFormat()) would otherwise get a mismatched view and fail to render.
+        hardwareTexture.format = texture.format;
+        hardwareTexture.originalFormat = texture.format;
         const internalTexture = new InternalTexture(this, InternalTextureSource.External, true);
         internalTexture._hardwareTexture = hardwareTexture;
         internalTexture.baseWidth = texture.width;

packages/dev/core/src/Materials/Textures/renderTargetTexture.pure.ts

@@ -1148,12 +1148,20 @@ export class RenderTargetTexture extends Texture implements IRenderTargetTexture
         return Math.min(FloorPOT(renderDimension), curved);
     }
 
+    /**
+     * The default array layer index bound by {@link _bindFrameBuffer} when no explicit layer is passed.
+     * Defaults to 0 (backward compatible). It is set per eye by the WebGPU XR layer provider so that each
+     * eye renders into its own layer of a layered projection-layer texture array.
+     * @internal
+     */
+    public _bindFrameBufferLayer = 0;
+
     /**
      * @internal
      * @param faceIndex face index to bind to if this is a cubetexture
      * @param layer defines the index of the texture to bind in the array
      */
-    public _bindFrameBuffer(faceIndex: number = 0, layer = 0) {
+    public _bindFrameBuffer(faceIndex: number = 0, layer = this._bindFrameBufferLayer) {
         const scene = this.getScene();
         if (!scene) {
             return;

packages/dev/core/src/XR/features/Layers/WebXRWebGPUCompositionLayer.ts

@@ -0,0 +1,136 @@
+import { type RenderTargetTexture } from "core/Materials/Textures/renderTargetTexture";
+import { type Viewport } from "core/Maths/math.viewport";
+import { Observable } from "core/Misc/observable";
+import { type WebXRLayerType, WebXRLayerWrapper } from "core/XR/webXRLayerWrapper";
+import { type WebXRLayerRenderTargetTextureProvider } from "core/XR/webXRRenderTargetTextureProvider";
+import { WebXRWebGPURenderTargetTextureProvider } from "core/XR/webXRWebGPURenderTargetTextureProvider";
+import { type WebXRSessionManager } from "core/XR/webXRSessionManager";
+import { type Nullable } from "core/types";
+
+/**
+ * Wraps xr composition layers for the WebGPU (XRGPUBinding) backend.
+ * Mirrors {@link WebXRCompositionLayerWrapper} for WebGPU.
+ * @internal
+ */
+export class WebXRWebGPUCompositionLayerWrapper extends WebXRLayerWrapper {
+    constructor(
+        public override getWidth: () => number,
+        public override getHeight: () => number,
+        public override readonly layer: XRCompositionLayer,
+        public override readonly layerType: WebXRLayerType,
+        public readonly isMultiview: boolean,
+        public createRTTProvider: (xrSessionManager: WebXRSessionManager) => WebXRLayerRenderTargetTextureProvider
+    ) {
+        super(getWidth, getHeight, layer, layerType, createRTTProvider);
+    }
+}
+
+/**
+ * Provides render target textures and other important rendering information for a given XRCompositionLayer
+ * on the WebGPU backend. Mirrors {@link WebXRCompositionLayerRenderTargetTextureProvider}, but wraps
+ * {@link XRGPUSubImage} GPUTextures instead of WebGL textures.
+ * @internal
+ */
+export class WebXRWebGPUCompositionLayerRenderTargetTextureProvider extends WebXRWebGPURenderTargetTextureProvider {
+    protected _lastSubImages = new Map<XREye, XRGPUSubImage>();
+    private _compositionLayer: XRCompositionLayer;
+    /**
+     * Fires every time a new render target texture is created (either for eye, for view, or for the entire frame)
+     */
+    public onRenderTargetTextureCreatedObservable = new Observable<{ texture: RenderTargetTexture; eye?: XREye }>();
+
+    constructor(
+        protected readonly _xrSessionManager: WebXRSessionManager,
+        protected readonly _xrGPUBinding: XRGPUBinding,
+        public override readonly layerWrapper: WebXRWebGPUCompositionLayerWrapper,
+        protected readonly _depthStencilFormat?: GPUTextureFormat
+    ) {
+        super(_xrSessionManager.scene, layerWrapper);
+        this._compositionLayer = layerWrapper.layer;
+    }
+
+    protected _getRenderTargetForSubImage(subImage: XRGPUSubImage, eye: XREye = "none") {
+        const lastSubImage = this._lastSubImages.get(eye);
+        const eyeIndex = eye == "right" ? 1 : 0;
+
+        const colorTexture = subImage.colorTexture;
+        const colorTextureWidth = colorTexture.width;
+        const colorTextureHeight = colorTexture.height;
+
+        const depthStencilTexture = subImage.depthStencilTexture ?? null;
+
+        const existingRenderTarget = this._renderTargetTextures[eyeIndex];
+        const sizeChanged = !existingRenderTarget || lastSubImage?.colorTexture.width !== colorTextureWidth || lastSubImage?.colorTexture.height !== colorTextureHeight;
+
+        if (sizeChanged) {
+            this._renderTargetTextures[eyeIndex] = this._createRenderTargetTextureFromGPUTextures(
+                colorTextureWidth,
+                colorTextureHeight,
+                colorTexture,
+                depthStencilTexture,
+                this._depthStencilFormat,
+                this.layerWrapper.isMultiview
+            );
+
+            this._framebufferDimensions = {
+                framebufferWidth: colorTextureWidth,
+                framebufferHeight: colorTextureHeight,
+            };
+            this.onRenderTargetTextureCreatedObservable.notifyObservers({ texture: this._renderTargetTextures[eyeIndex], eye });
+        } else {
+            // Same size: repoint the wrapped textures at this frame's GPUTextures, preserving the
+            // RenderTargetTexture / InternalTexture identity held by the XR camera's outputRenderTarget.
+            this._updateRenderTargetTextureFromGPUTextures(existingRenderTarget, colorTexture, depthStencilTexture);
+        }
+
+        this._lastSubImages.set(eye, subImage);
+
+        // The projection-layer color/depth textures may be a texture ARRAY with one layer per eye
+        // (depthOrArrayLayers > 1). The sub-image's view descriptor carries the authoritative array-layer
+        // index for this eye, so route it into the render target's bind so each eye renders into its own
+        // layer. For non-layered textures this is 0 (unchanged behavior).
+        const renderTargetTexture = this._renderTargetTextures[eyeIndex];
+        renderTargetTexture._bindFrameBufferLayer = subImage.getViewDescriptor().baseArrayLayer ?? 0;
+
+        return renderTargetTexture;
+    }
+
+    private _getSubImageForEye(eye?: XREye): Nullable<XRGPUSubImage> {
+        const currentFrame = this._xrSessionManager.currentFrame;
+        if (currentFrame) {
+            return this._xrGPUBinding.getSubImage(this._compositionLayer, currentFrame, eye);
+        }
+        return null;
+    }
+
+    public getRenderTargetTextureForEye(eye?: XREye): Nullable<RenderTargetTexture> {
+        const subImage = this._getSubImageForEye(eye);
+        if (subImage) {
+            return this._getRenderTargetForSubImage(subImage, eye);
+        }
+        return null;
+    }
+
+    public getRenderTargetTextureForView(view?: XRView): Nullable<RenderTargetTexture> {
+        return this.getRenderTargetTextureForEye(view?.eye);
+    }
+
+    protected _setViewportForSubImage(viewport: Viewport, subImage: XRGPUSubImage) {
+        const textureWidth = subImage.colorTexture.width;
+        const textureHeight = subImage.colorTexture.height;
+        const xrViewport = subImage.viewport;
+        viewport.x = xrViewport.x / textureWidth;
+        viewport.y = xrViewport.y / textureHeight;
+        viewport.width = xrViewport.width / textureWidth;
+        viewport.height = xrViewport.height / textureHeight;
+    }
+
+    public trySetViewportForView(viewport: Viewport, view: XRView): boolean {
+        const subImage = this._lastSubImages.get(view.eye) || this._getSubImageForEye(view.eye);
+        if (subImage) {
+            this._setViewportForSubImage(viewport, subImage);
+            return true;
+        }
+        return false;
+    }
+}

packages/dev/core/src/XR/features/Layers/WebXRWebGPUProjectionLayer.ts

@@ -0,0 +1,100 @@
+import { type WebXRSessionManager } from "core/XR/webXRSessionManager";
+import { WebXRWebGPUCompositionLayerRenderTargetTextureProvider, WebXRWebGPUCompositionLayerWrapper } from "./WebXRWebGPUCompositionLayer";
+import { type Nullable } from "core/types";
+import { type RenderTargetTexture } from "core/Materials/Textures/renderTargetTexture";
+import { type Viewport } from "core/Maths/math.viewport";
+
+/**
+ * Wraps xr projection layers for the WebGPU (XRGPUBinding) backend.
+ * Mirrors {@link WebXRProjectionLayerWrapper} for WebGPU.
+ * @internal
+ */
+export class WebXRWebGPUProjectionLayerWrapper extends WebXRWebGPUCompositionLayerWrapper {
+    constructor(
+        public override readonly layer: XRProjectionLayer,
+        isMultiview: boolean,
+        xrGPUBinding: XRGPUBinding,
+        depthStencilFormat?: GPUTextureFormat
+    ) {
+        super(
+            () => layer.textureWidth,
+            () => layer.textureHeight,
+            layer,
+            "XRProjectionLayer",
+            isMultiview,
+            (sessionManager) => new WebXRWebGPUProjectionLayerRenderTargetTextureProvider(sessionManager, xrGPUBinding, this, depthStencilFormat)
+        );
+    }
+}
+
+/**
+ * Provides render target textures and other important rendering information for a given XRProjectionLayer
+ * on the WebGPU backend.
+ * @internal
+ */
+class WebXRWebGPUProjectionLayerRenderTargetTextureProvider extends WebXRWebGPUCompositionLayerRenderTargetTextureProvider {
+    private readonly _projectionLayer: XRProjectionLayer;
+
+    constructor(
+        _xrSessionManager: WebXRSessionManager,
+        _xrGPUBinding: XRGPUBinding,
+        public override readonly layerWrapper: WebXRWebGPUProjectionLayerWrapper,
+        depthStencilFormat?: GPUTextureFormat
+    ) {
+        super(_xrSessionManager, _xrGPUBinding, layerWrapper, depthStencilFormat);
+        this._projectionLayer = layerWrapper.layer;
+    }
+
+    private _getSubImageForView(view: XRView): XRGPUSubImage {
+        return this._xrGPUBinding.getViewSubImage(this._projectionLayer, view);
+    }
+
+    public override getRenderTargetTextureForView(view: XRView): Nullable<RenderTargetTexture> {
+        return this._getRenderTargetForSubImage(this._getSubImageForView(view), view.eye);
+    }
+
+    public override getRenderTargetTextureForEye(eye: XREye): Nullable<RenderTargetTexture> {
+        const lastSubImage = this._lastSubImages.get(eye);
+        if (lastSubImage) {
+            return this._getRenderTargetForSubImage(lastSubImage, eye);
+        }
+        return null;
+    }
+
+    public override trySetViewportForView(viewport: Viewport, view: XRView): boolean {
+        const subImage = this._lastSubImages.get(view.eye) || this._getSubImageForView(view);
+        if (subImage) {
+            this._setViewportForSubImage(viewport, subImage);
+            return true;
+        }
+        return false;
+    }
+}
+
+/**
+ * The default depth/stencil format used for a WebGPU projection layer.
+ * Mirrors the WebGL default (DEPTH24_STENCIL8).
+ * @internal
+ */
+export const DefaultXRGPUProjectionLayerDepthStencilFormat: GPUTextureFormat = "depth24plus-stencil8";
+
+/**
+ * Builds the default XRGPUProjectionLayerInit for a WebGPU projection layer.
+ * The color format must be the binding's preferred color format, so it is provided by the caller.
+ * @param colorFormat the preferred color format reported by the XRGPUBinding
+ * @param depthStencilFormat the depth/stencil format to request (defaults to depth24plus-stencil8)
+ * @returns the projection layer init to pass to XRGPUBinding.createProjectionLayer
+ * @internal
+ */
+export function CreateDefaultXRGPUProjectionLayerInit(
+    colorFormat: GPUTextureFormat,
+    depthStencilFormat: GPUTextureFormat = DefaultXRGPUProjectionLayerDepthStencilFormat
+): XRGPUProjectionLayerInit {
+    return {
+        colorFormat,
+        depthStencilFormat,
+        // GPUTextureUsage.RENDER_ATTACHMENT (0x10) — the spec default, stated explicitly here.
+        textureUsage: 0x10,
+        scaleFactor: 1.0,
+    };
+}

packages/dev/core/src/XR/features/WebXRLayers.pure.ts

@@ -7,6 +7,8 @@ import { type WebXRLayerWrapper } from "../webXRLayerWrapper";
 import { WebXRWebGLLayerWrapper } from "../webXRWebGLLayer";
 import { WebXRProjectionLayerWrapper, DefaultXRProjectionLayerInit } from "./Layers/WebXRProjectionLayer";
 import { WebXRCompositionLayerRenderTargetTextureProvider, WebXRCompositionLayerWrapper } from "./Layers/WebXRCompositionLayer";
+import { WebXRWebGPUProjectionLayerWrapper, CreateDefaultXRGPUProjectionLayerInit } from "./Layers/WebXRWebGPUProjectionLayer";
+import { WebXRGraphicsBindingType, type WebXRWebGPUGraphicsBinding } from "../webXRGraphicsBinding";
 import { type ThinTexture } from "../../Materials/Textures/thinTexture";
 import { type DynamicTexture } from "../../Materials/Textures/dynamicTexture.pure";
 import { Color4 } from "../../Maths/math.color.pure";
@@ -52,6 +54,8 @@ export class WebXRLayers extends WebXRAbstractFeature {
 
     private _glContext: WebGLRenderingContext | WebGL2RenderingContext;
     private _xrWebGLBinding: XRWebGLBinding;
+    private _isWebGPU = false;
+    private _xrGPUBinding?: XRGPUBinding;
     private _isMultiviewEnabled = false;
     private _projectionLayerInitialized = false;
 
@@ -78,13 +82,26 @@ export class WebXRLayers extends WebXRAbstractFeature {
         }
 
         const engine = this._xrSessionManager.scene.getEngine();
-        this._glContext = (engine as ThinEngine)._gl;
-        this._xrWebGLBinding = new XRWebGLBinding(this._xrSessionManager.session, this._glContext);
         this._existingLayers.length = 0;
+        this._isWebGPU = engine.isWebGPU;
 
-        const projectionLayerInit = { ...DefaultXRProjectionLayerInit, ...this._options.projectionLayerInit };
-        this._isMultiviewEnabled = this._options.preferMultiviewOnInit && engine.getCaps().multiview;
-        this.createProjectionLayer(projectionLayerInit /*, projectionLayerMultiview*/);
+        if (this._isWebGPU) {
+            const binding = this._xrSessionManager._getGraphicsBinding();
+            if (binding.bindingType !== WebXRGraphicsBindingType.WebGPU) {
+                throw new Error("Expected a WebGPU graphics binding for a WebGPU engine.");
+            }
+            this._xrGPUBinding = (binding as WebXRWebGPUGraphicsBinding).binding;
+            // Multiview is not yet supported on the WebGPU XR path; force single-view (two sub-images).
+            this._isMultiviewEnabled = false;
+            this._createWebGPUProjectionLayer();
+        } else {
+            this._glContext = (engine as ThinEngine)._gl;
+            this._xrWebGLBinding = new XRWebGLBinding(this._xrSessionManager.session, this._glContext);
+
+            const projectionLayerInit = { ...DefaultXRProjectionLayerInit, ...this._options.projectionLayerInit };
+            this._isMultiviewEnabled = this._options.preferMultiviewOnInit && engine.getCaps().multiview;
+            this.createProjectionLayer(projectionLayerInit /*, projectionLayerMultiview*/);
+        }
         this._projectionLayerInitialized = true;
 
         return true;
@@ -150,6 +167,23 @@ export class WebXRLayers extends WebXRAbstractFeature {
         return layer;
     }
 
+    /**
+     * Creates the base projection layer for the WebGPU (XRGPUBinding) backend.
+     * Single-view only for now (multiview is deferred); the color format is the binding's preferred format.
+     * @returns the WebGPU projection layer wrapper
+     */
+    private _createWebGPUProjectionLayer(): WebXRWebGPUProjectionLayerWrapper {
+        if (!this._xrSessionManager.inXRSession) {
+            throw new Error("Cannot create a layer outside of a WebXR session. Make sure the session has started before creating layers.");
+        }
+        const binding = this._xrGPUBinding!;
+        const init = CreateDefaultXRGPUProjectionLayerInit(binding.getPreferredColorFormat());
+        const projLayer = binding.createProjectionLayer(init);
+        const layer = new WebXRWebGPUProjectionLayerWrapper(projLayer, false, binding, init.depthStencilFormat);
+        this.addXRSessionLayer(layer);
+        return layer;
+    }
+
     /**
      * Note about making it private - this function will be exposed once I decide on a proper API to support all of the XR layers' options
      * @param options an object providing configuration options for the new XRQuadLayer.
@@ -335,7 +369,13 @@ export class WebXRLayers extends WebXRAbstractFeature {
 
     public override isCompatible(): boolean {
         // TODO (rgerd): Add native support.
-        return !this._xrSessionManager.isNative && typeof XRWebGLBinding !== "undefined" && !!XRWebGLBinding.prototype.createProjectionLayer;
+        if (this._xrSessionManager.isNative) {
+            return false;
+        }
+        if (this._xrSessionManager.scene.getEngine().isWebGPU) {
+            return typeof XRGPUBinding !== "undefined" && !!XRGPUBinding.prototype.createProjectionLayer;
+        }
+        return typeof XRWebGLBinding !== "undefined" && !!XRWebGLBinding.prototype.createProjectionLayer;
     }
 
     /**