refactor normal picking to use MRT instead, removed multiple sampling normal derivatives.

Author: MAG-AdrianMeredith

examples/src/examples/misc/editor.selector.mjs

@@ -43,7 +43,7 @@ class Selector extends pc.EventHandler {
         this._scene = app.scene;
         const device = app.graphicsDevice;
         // depth enabled so we can also recover the world point + surface normal at the click
-        this._picker = new pc.Picker(app, device.canvas.width, device.canvas.height, true);
+        this._picker = new pc.Picker(app, device.canvas.width, device.canvas.height, true, true);
         this._layers = layers;
 
         this._onPointerDown = this._onPointerDown.bind(this);

src/framework/graphics/picker.js

@@ -126,6 +126,23 @@ class Picker {
      */
     renderTargetDepth = null;
 
+    /**
+     * Optional buffer holding the world-space surface normal (RGBA8, encoded as n*0.5+0.5) for
+     * normal picking.
+     *
+     * @type {Texture|null}
+     * @private
+     */
+    normalBuffer = null;
+
+    /**
+     * Internal render target for reading the normal buffer.
+     *
+     * @type {RenderTarget|null}
+     * @private
+     */
+    renderTargetNormal = null;
+
     /**
      * Mapping table from ids to MeshInstances or GSplatComponents.
      *
@@ -149,16 +166,22 @@ class Picker {
      * @param {number} height - The height of the pick buffer in pixels.
      * @param {boolean} [depth] - Whether to enable depth picking. When enabled, depth
      * information is captured alongside mesh IDs using MRT. Defaults to false.
+     * @param {boolean} [normals] - Whether to enable normal picking. When enabled, the world-space
+     * surface normal is captured alongside mesh IDs and depth using MRT, letting
+     * {@link Picker#getWorldPointAndNormalAsync} read the normal directly instead of
+     * reconstructing it from neighboring depths. Implies depth. Defaults to false.
      */
-    constructor(app, width, height, depth = false) {
+    constructor(app, width, height, depth = false, normals = false) {
         // Note: The only reason this class needs the app is to access the renderer. Ideally we remove this dependency and move
         // the Picker from framework to the scene level, or even the extras.
         Debug.assert(app);
         this.device = app.graphicsDevice;
 
         this.renderPass = new RenderPassPicker(this.device, app.renderer);
 
-        this.depth = depth;
+        this.normals = normals;
+        // normal picking captures depth too (needed to reconstruct the world point)
+        this.depth = depth || normals;
         this.width = 0;
         this.height = 0;
         this.resize(width, height);
@@ -308,18 +331,15 @@ class Picker {
     }
 
     /**
-     * Return the world position and a derived surface normal at the specified screen coordinates.
-     * The normal is computed from depth-buffer neighbors via a cross product of two screen-aligned
-     * world-space tangent vectors — it is a faceted (per-pixel) normal, not an interpolated vertex
-     * normal. Internally a single 2x2 depth readback is used and {@link Picker#prepare} auto-pads
-     * any supplied scissor rect by 1 pixel so the neighbor samples are inside the rasterized
-     * region, so callers do not need to widen the scissor themselves.
+     * Return the world position and surface normal at the specified screen coordinates. Requires
+     * the picker to have been created with `normals: true`: the world-space surface normal is read
+     * directly from its dedicated MRT attachment (a single-pixel read), and the world point is
+     * reconstructed from the depth attachment at the same pixel.
      *
      * @param {number} x - The x coordinate of the pixel to pick.
      * @param {number} y - The y coordinate of the pixel to pick.
-     * @returns {Promise<{point: Vec3, normal: Vec3}|null>} Promise resolving to the world point
-     * and a unit normal facing the camera, or null if depth picking is not enabled or any of the
-     * required neighbor pixels did not hit an object.
+     * @returns {Promise<{point: Vec3, normal: Vec3}|null>} Promise resolving to the world point and
+     * a unit world-space normal, or null if normal picking is not enabled or nothing was picked.
      * @example
      * picker.prepare(camera, scene, layers, { x: px, y: py, width: 1, height: 1 });
      * picker.getWorldPointAndNormalAsync(px, py).then((hit) => {
@@ -329,65 +349,27 @@ class Picker {
      * });
      */
     async getWorldPointAndNormalAsync(x, y) {
-        if (!this.depthBuffer) {
+        if (!this.normalBuffer) {
             return null;
         }
         const state = this._captureCameraState();
         if (!state) {
             return null;
         }
 
-        // one 2x2 readback covering (x,y), (x+1,y), (x,y+1), (x+1,y+1)
-        const pixels = await this._readTexture(this.depthBuffer, x, y, 2, 2, /** @type {RenderTarget} */ (this.renderTargetDepth));
-
-        // decode 4 RGBA8 pixels → 4 linear depths (null if cleared)
-        /** @type {(number|null)[]} */
-        const depths = [null, null, null, null];
-        for (let i = 0; i < 4; i++) {
-            const o = i * 4;
-            const intBits = ((pixels[o] << 24) | (pixels[o + 1] << 16) | (pixels[o + 2] << 8) | pixels[o + 3]) >>> 0;
-            if (intBits !== 0xFFFFFFFF) {
-                _int32View[0] = intBits;
-                depths[i] = _floatView[0];
-            }
-        }
-
-        // map array indices to screen positions. WebGPU returns rows top-down; WebGL2 returns
-        // them bottom-up of the (already Y-flipped) read region — which means row 0 corresponds
-        // to the BOTTOM screen row of the original (top-left origin) request.
-        let dCenter, dRight, dDown;
-        if (this.device.isWebGPU) {
-            // pixels[0]=(x,y), [1]=(x+1,y), [2]=(x,y+1), [3]=(x+1,y+1)
-            dCenter = depths[0];
-            dRight = depths[1];
-            dDown = depths[2];
-        } else {
-            // pixels[0]=(x,y+1), [1]=(x+1,y+1), [2]=(x,y), [3]=(x+1,y)
-            dDown = depths[0];
-            dCenter = depths[2];
-            dRight = depths[3];
-        }
-        if (dCenter === null || dRight === null || dDown === null) {
+        // depth gates the hit: the normal buffer is cleared to white, which would otherwise decode
+        // to a valid-looking normal, so a null depth means "nothing picked here".
+        const depth = await this.getPointDepthAsync(x, y);
+        if (depth === null) {
             return null;
         }
 
-        const pCenter = this._unprojectDepth(x, y, dCenter, state);
-        const pRight = this._unprojectDepth(x + 1, y, dRight, state);
-        const pDown = this._unprojectDepth(x, y + 1, dDown, state);
-
-        // tangent vectors along screen +x and screen +y, then face normal via cross product
-        const vRight = new Vec3().sub2(pRight, pCenter);
-        const vDown = new Vec3().sub2(pDown, pCenter);
-        const normal = new Vec3().cross(vRight, vDown).normalize();
-
-        // ensure the normal points back toward the camera
-        const camPos = state.cameraPos;
-        const toCam = new Vec3().sub2(camPos, pCenter);
-        if (normal.dot(toCam) < 0) {
-            normal.mulScalar(-1);
+        const normal = await this._readNormalAt(x, y);
+        if (!normal) {
+            return null;
         }
 
-        return { point: pCenter, normal: normal };
+        return { point: this._unprojectDepth(x, y, depth, state), normal };
     }
 
     /**
@@ -449,6 +431,7 @@ class Picker {
      * @ignore
      */
     async getPointDepthAsync(x, y) {
+
         if (!this.depthBuffer) {
             return null;
         }
@@ -468,6 +451,39 @@ class Picker {
         return _floatView[0];
     }
 
+    /**
+     * Read and decode the world-space surface normal at the given pick-buffer pixel from the
+     * normal attachment (RGBA8, encoded as n*0.5+0.5). Callers should gate on a valid depth at the
+     * same pixel, since the buffer is cleared to white and a cleared pixel decodes to a non-null
+     * (but meaningless) normal.
+     *
+     * @param {number} x - Pick-buffer x in pixels, top-left origin.
+     * @param {number} y - Pick-buffer y in pixels, top-left origin.
+     * @returns {Promise<Vec3|null>} The unit world-space normal, or null if normal picking is not
+     * enabled or the sample is degenerate.
+     * @private
+     */
+    async _readNormalAt(x, y) {
+        if (!this.normalBuffer) {
+            return null;
+        }
+
+        const pixels = await this._readTexture(this.normalBuffer, x, y, 1, 1, /** @type {RenderTarget} */ (this.renderTargetNormal));
+
+        // decode RGBA8 (n*0.5+0.5) back to a world-space vector in [-1, 1]
+        const normal = new Vec3(
+            (pixels[0] / 255) * 2 - 1,
+            (pixels[1] / 255) * 2 - 1,
+            (pixels[2] / 255) * 2 - 1
+        );
+
+        const len = normal.length();
+        if (len < 1e-4) {
+            return null;
+        }
+        return normal.mulScalar(1 / len);
+    }
+
     // sanitize the rectangle to make sure it's inside the texture and does not use fractions
     sanitizeRect(x, y, width, height) {
         const maxWidth = this.renderTarget.width;
@@ -537,6 +553,18 @@ class Picker {
             });
         }
 
+        if (this.normals) {
+            // create normal buffer for MRT (attachment 2, matches pcFragColor2)
+            this.normalBuffer = this.createTexture('pick-normal');
+            colorBuffers.push(this.normalBuffer);
+
+            // create a render target for reading the normal buffer
+            this.renderTargetNormal = new RenderTarget({
+                colorBuffer: this.normalBuffer,
+                depth: false
+            });
+        }
+
         this.renderTarget = new RenderTarget({
             colorBuffers: colorBuffers,
             depth: true
@@ -551,8 +579,12 @@ class Picker {
         this.renderTargetDepth?.destroy();
         this.renderTargetDepth = null;
 
+        this.renderTargetNormal?.destroy();
+        this.renderTargetNormal = null;
+
         this.colorBuffer = null;
         this.depthBuffer = null;
+        this.normalBuffer = null;
     }
 
     /**
@@ -580,6 +612,7 @@ class Picker {
         // make the render target the right size
         this.renderTarget?.resize(this.width, this.height);
         this.renderTargetDepth?.resize(this.width, this.height);
+        this.renderTargetNormal?.resize(this.width, this.height);
 
         // clear registered meshes mapping
         this.mapping.clear();
@@ -591,22 +624,23 @@ class Picker {
         renderPass.setClearColor(Color.WHITE);
         renderPass.depthStencilOps.clearDepth = true;
 
-        // optional scissor rect — sanitize and flip top-left → bottom-left for device.setScissor.
-        // Pad width/height by 1 so neighbor pixels needed by getWorldPointAndNormalAsync
-        // (which reads a 2x2 depth block) stay inside the rasterized region.
+        // optional scissor rect — sanitize and flip top-left → bottom-left for device.setScissor
         let scissorRect = null;
         if (options && this.renderTarget) {
             const w = options.width ?? 0;
             const h = options.height ?? 0;
             if (w > 0 && h > 0) {
-                const r = this.sanitizeRect(options.x ?? 0, options.y ?? 0, w + 1, h + 1);
+                const r = this.sanitizeRect(options.x ?? 0, options.y ?? 0, w, h);
                 const flippedY = this.renderTarget.height - (r.y + r.w);
                 scissorRect = new Vec4(r.x, flippedY, r.z, r.w);
             }
         }
 
+        const writeDepth = this.depth;
+        const writeNormals = this.normals;
+
         // render the pass to update the render target
-        renderPass.update(camera, scene, layers, this.mapping, this.depth, scissorRect);
+        renderPass.update(camera, scene, layers, this.mapping, writeDepth, writeNormals, scissorRect);
         renderPass.render();
     }
 

src/framework/graphics/render-pass-picker.js

@@ -1,7 +1,7 @@
 import { DebugGraphics } from '../../platform/graphics/debug-graphics.js';
 import { BlendState } from '../../platform/graphics/blend-state.js';
 import { RenderPass } from '../../platform/graphics/render-pass.js';
-import { SHADER_PICK, SHADER_DEPTH_PICK } from '../../scene/constants.js';
+import { SHADER_PICK, SHADER_DEPTH_PICK, SHADER_NORMAL_PICK } from '../../scene/constants.js';
 
 /**
  * @import { BindGroup } from '../../platform/graphics/bind-group.js'
@@ -44,6 +44,9 @@ class RenderPassPicker extends RenderPass {
     /** @type {boolean} */
     depth;
 
+    /** @type {boolean} */
+    normals;
+
     /** @type {Vec4|null} Optional scissor rect in render-target pixel coords, bottom-left origin. */
     scissorRect = null;
 
@@ -72,16 +75,18 @@ class RenderPassPicker extends RenderPass {
      * @param {Layer[]} layers - The layers to pick from.
      * @param {Map<number, MeshInstance | GSplatComponent>} mapping - Map to store ID to object mappings.
      * @param {boolean} depth - Whether to render depth information.
+     * @param {boolean} normals - Whether to render the world-space surface normal (implies depth).
      * @param {Vec4|null} [scissorRect] - Optional scissor rect in render-target pixel coords,
      * bottom-left origin (x, y, w, h). If set, only fragments inside the rect are rasterized;
      * outside the rect the cleared "no selection" value (white) remains.
      */
-    update(camera, scene, layers, mapping, depth, scissorRect = null) {
+    update(camera, scene, layers, mapping, depth, normals, scissorRect = null) {
         this.camera = camera;
         this.scene = scene;
         this.layers = layers;
         this.mapping = mapping;
         this.depth = depth;
+        this.normals = normals;
         this.scissorRect = scissorRect;
 
         if (scene.clusteredLightingEnabled) {
@@ -207,7 +212,7 @@ class RenderPassPicker extends RenderPass {
                     renderer.setupViewUniformBuffers(this.viewBindGroups, renderer.viewUniformFormat, renderer.viewBindGroupFormat, null);
                 }
 
-                const shaderPass = this.depth ? SHADER_DEPTH_PICK : SHADER_PICK;
+                const shaderPass = this.normals ? SHADER_NORMAL_PICK : (this.depth ? SHADER_DEPTH_PICK : SHADER_PICK);
                 renderer.renderForward(camera.camera, renderTarget, tempMeshInstances, lights, shaderPass, (meshInstance) => {
                     device.setBlendState(this.blendState);
                 });

src/scene/constants.js

@@ -815,6 +815,9 @@ export const SHADER_PICK = 3;
 // shader pass used by the Picker class to render mesh ID and depth
 export const SHADER_DEPTH_PICK = 4;
 
+// shader pass used by the Picker class to render mesh ID, depth and world-space surface normal
+export const SHADER_NORMAL_PICK = 5;
+
 /**
  * Shader that performs forward rendering.
  *

src/scene/shader-lib/glsl/chunks/common/frag/pick.js

@@ -31,4 +31,13 @@ vec4 encodePickOutput(uint id) {
         return float2uint(linearDepth);
     }
 #endif
+
+#ifdef NORMAL_PICK_PASS
+    // Encode a world-space surface normal ([-1, 1]) into RGBA8 ([0, 1]). Alpha is set to 1 as a
+    // hit marker, though readback should gate on the depth attachment (the buffer is cleared to
+    // white, so alpha alone can't distinguish a hit from the cleared background).
+    vec4 getPickNormal(vec3 worldNormal) {
+        return vec4(normalize(worldNormal) * 0.5 + 0.5, 1.0);
+    }
+#endif
 `;

src/scene/shader-lib/glsl/chunks/lit/frag/pass-other/litOtherMain.js

@@ -20,6 +20,13 @@ void main(void) {
         #ifdef DEPTH_PICK_PASS
             pcFragColor1 = getPickDepth();
         #endif
+        #ifdef NORMAL_PICK_PASS
+            // world-space interpolated vertex normal. The normal-pick pass forces needsNormal,
+            // so the NORMALS varying (vNormalW) is always generated. getPickNormal normalizes,
+            // so the raw interpolated value is fine. Note: this is the geometric surface normal,
+            // not the normal-mapped one - sufficient for pick-point orientation.
+            pcFragColor2 = getPickNormal(vNormalW);
+        #endif
     #endif
 
     #ifdef PREPASS_PASS

src/scene/shader-lib/programs/lit-shader.js

@@ -9,7 +9,7 @@ import {
 import {
     LIGHTSHAPE_PUNCTUAL,
     LIGHTTYPE_DIRECTIONAL, LIGHTTYPE_OMNI, LIGHTTYPE_SPOT,
-    SHADER_PICK,
+    SHADER_PICK, SHADER_NORMAL_PICK,
     SPRITE_RENDERMODE_SLICED, SPRITE_RENDERMODE_TILED, shadowTypeInfo, SHADER_PREPASS,
     lightTypeNames, lightShapeNames, spriteRenderModeNames, fresnelNames, blendNames, lightFalloffNames,
     cubemaProjectionNames, specularOcclusionNames, reflectionSrcNames, ambientSrcNames,
@@ -166,6 +166,9 @@ class LitShader {
             (options.clusteredLightingEnabled && !this.shadowPass) ||
             options.useClearCoatNormals;
         this.needsNormal = this.needsNormal && !this.shadowPass;
+        // the normal-pick pass writes the world-space surface normal (dNormalW) to an MRT target,
+        // so the frontend must compute it even though this pass does no lighting
+        this.needsNormal = this.needsNormal || options.pass === SHADER_NORMAL_PICK;
         this.needsSceneColor = options.useDynamicRefraction;
         this.needsScreenSize = options.useDynamicRefraction;
         this.needsTransforms = options.useDynamicRefraction;

src/scene/shader-lib/wgsl/chunks/common/frag/pick.js

@@ -31,4 +31,13 @@ fn encodePickOutput(id: u32) -> vec4f {
         return float2uint(linearDepth);
     }
 #endif
+
+#ifdef NORMAL_PICK_PASS
+    // Encode a world-space surface normal ([-1, 1]) into RGBA8 ([0, 1]). Alpha is set to 1 as a
+    // hit marker, though readback should gate on the depth attachment (the buffer is cleared to
+    // white, so alpha alone can't distinguish a hit from the cleared background).
+    fn getPickNormal(worldNormal: vec3f) -> vec4f {
+        return vec4f(normalize(worldNormal) * 0.5 + 0.5, 1.0);
+    }
+#endif
 `;