3dgs-offscreen: RenderToImage/RenderToDepthImage with GPU depth merge

gaussian_depth_merge.comp:
- New compute shader: merges GS composite depth (R32F) and Filament scene
  depth (GL_DEPTH_COMPONENT32F, reversed-Z) into a normalised R16UI texture
- Per-pixel min-depth merge in linear space; outputs [0,65535] uint16
- Bound at composite stage binding 14 (reuses radix UBO slot which is idle)

FilamentRenderToBuffer::Render():
- Mirror FilamentRenderer interactive pipeline for offscreen captures:
  BeginFrame -> flushAndWait -> RenderGeometryStage -> render() ->
  flushAndWait -> RenderCompositeStage
- Color readback: two parallel readPixels (Filament RGBA+UBYTE base,
  GS RGBA+FLOAT overlay) then CPU BlendPremultipliedSplatOverRgb8()
- Depth readback: ReadMergedDepthToUint16Cpu() reads the R16UI merged
  texture via glGetTexImage; converts uint16 -> float [0,1] for the
  RenderToDepthImage callback; fallback to Filament-only depth if GS
  depth unavailable
- Metal: readPixels always uses RGBA+UBYTE (no native RGB Metal format);
  alpha stripped when n_channels_==3
- EnableViewCaching(true) in SetDimensions when GS geometry present
  (required for render-target-based readPixels)

Author: ssheorey

cpp/open3d/visualization/rendering/filament/FilamentRenderToBuffer.cpp

@@ -32,16 +32,57 @@
 #pragma warning(pop)
 #endif  // _MSC_VER
 
+#include <algorithm>
+#include <cstdint>
+#include <vector>
+
 #include "open3d/utility/Logging.h"
 #include "open3d/visualization/rendering/filament/FilamentEngine.h"
 #include "open3d/visualization/rendering/filament/FilamentRenderer.h"
+#include "open3d/visualization/rendering/filament/FilamentResourceManager.h"
 #include "open3d/visualization/rendering/filament/FilamentScene.h"
 #include "open3d/visualization/rendering/filament/FilamentView.h"
+#include "open3d/visualization/rendering/gaussian_splat/GaussianSplatRenderer.h"
 
 namespace open3d {
 namespace visualization {
 namespace rendering {
 
+namespace {
+
+/// Composite shader stores premultiplied RGB in \p gs_rgba; blend like ImGui
+/// \c One / \c OneMinusSrcAlpha over an opaque Filament base.
+void BlendPremultipliedSplatOverRgb8(
+        uint8_t* base_rgb, int n_channels, const float* gs_rgba, int w, int h) {
+    const int n = w * h;
+    for (int i = 0; i < n; ++i) {
+        const float fr = gs_rgba[i * 4 + 0];
+        const float fg = gs_rgba[i * 4 + 1];
+        const float fb = gs_rgba[i * 4 + 2];
+        const float fa = gs_rgba[i * 4 + 3];
+        const float br = static_cast<float>(base_rgb[i * n_channels + 0]) *
+                         (1.f / 255.f);
+        const float bg = static_cast<float>(base_rgb[i * n_channels + 1]) *
+                         (1.f / 255.f);
+        const float bb = static_cast<float>(base_rgb[i * n_channels + 2]) *
+                         (1.f / 255.f);
+        const float r = fr + br * (1.f - fa);
+        const float g = fg + bg * (1.f - fa);
+        const float b = fb + bb * (1.f - fa);
+        base_rgb[i * n_channels + 0] = static_cast<uint8_t>(
+                std::min(std::max(r, 0.f), 1.f) * 255.f + 0.5f);
+        base_rgb[i * n_channels + 1] = static_cast<uint8_t>(
+                std::min(std::max(g, 0.f), 1.f) * 255.f + 0.5f);
+        base_rgb[i * n_channels + 2] = static_cast<uint8_t>(
+                std::min(std::max(b, 0.f), 1.f) * 255.f + 0.5f);
+        if (n_channels == 4) {
+            base_rgb[i * n_channels + 3] = 255;
+        }
+    }
+}
+
+}  // namespace
+
 FilamentRenderToBuffer::FilamentRenderToBuffer(filament::Engine& engine)
     : engine_(engine) {
     renderer_ = engine_.createRenderer();
@@ -121,6 +162,12 @@ void FilamentRenderToBuffer::SetDimensions(const std::uint32_t width,
     width_ = width;
     height_ = height;
 
+    // Allocate cached Filament color/depth attachments for Gaussian splat
+    // zero-copy and for readPixels of the Filament base pass.
+    if (scene_ && scene_->HasGaussianSplatGeometry()) {
+        view_->EnableViewCaching(true);
+    }
+
     if (depth_image_) {
         buffer_size_ = width * height * sizeof(std::float_t);
     } else {
@@ -173,31 +220,221 @@ void FilamentRenderToBuffer::ReadPixelsCallback(void*, size_t, void* user) {
     delete params;
 }
 
+// Ordering mirrors FilamentRenderer::{BeginFrame,Draw,EndFrame}.
+// Stage 1 (Geometry) runs before Filament's beginFrame.
+// Stage 2 (Composite) runs after render() on non-Apple, after endFrame() on
+// Apple.
 void FilamentRenderToBuffer::Render() {
     frame_done_ = false;
     scene_->HideRefractedMaterials();
+
+    const bool has_gaussian =
+            gaussian_splat_renderer_ && scene_->HasGaussianSplatGeometry();
+    const bool run_gs_pipeline = has_gaussian;
+
+    if (run_gs_pipeline) {
+        gaussian_splat_renderer_->RequestRedrawForView(*view_);
+        if (depth_image_) {
+            // Signal that a depth readback is needed so the composite pass
+            // allocates and populates the merged_depth_u16_tex scratch texture.
+            gaussian_splat_renderer_->RequestDepthReadbackForView(*view_, true);
+        }
+        gaussian_splat_renderer_->BeginFrame();
+#if !defined(__APPLE__)
+        // Drain Filament work before Gaussian compute dispatches (shared
+        // GL/Vulkan queue on non-Apple backends).
+        engine_.flushAndWait();
+#endif
+        gaussian_splat_renderer_->RenderGeometryStage(*view_, *scene_);
+    }
+
     if (renderer_->beginFrame(swapchain_)) {
         renderer_->render(view_->GetNativeView());
 
+#if !defined(__APPLE__)
+        if (run_gs_pipeline) {
+            engine_.flushAndWait();
+            gaussian_splat_renderer_->RenderCompositeStage(*view_);
+        }
+#endif
+
         using namespace filament;
         using namespace backend;
 
-        auto format = (n_channels_ == 3 ? PixelDataFormat::RGB
-                                        : PixelDataFormat::RGBA);
-        auto type = PixelDataType::UBYTE;
-        if (depth_image_) {
-            format = PixelDataFormat::DEPTH_COMPONENT;
-            type = PixelDataType::FLOAT;
-        }
-        auto user_param = new PBDParams(this, callback_);
-        PixelBufferDescriptor pd(buffer_, buffer_size_, format, type,
-                                 ReadPixelsCallback, user_param);
         auto vp = view_->GetNativeView()->getViewport();
 
-        renderer_->readPixels(vp.left, vp.bottom, vp.width, vp.height,
-                              std::move(pd));
-
         renderer_->endFrame();
+
+#if defined(__APPLE__)
+        if (run_gs_pipeline) {
+            gaussian_splat_renderer_->RenderCompositeStage(*view_);
+        }
+#endif
+
+        engine_.flushAndWait();
+
+        auto* resource_mgr = &EngineInstance::GetResourceManager();
+
+        RenderTargetHandle view_rt_h = view_->GetRenderTargetHandle();
+        filament::RenderTarget* native_view_rt = nullptr;
+        if (view_rt_h) {
+            auto weak_vrt = resource_mgr->GetRenderTarget(view_rt_h);
+            if (auto vrt = weak_vrt.lock()) {
+                native_view_rt = vrt.get();
+            }
+        }
+
+        RenderTargetHandle gs_rt =
+                run_gs_pipeline
+                        ? gaussian_splat_renderer_->GetColorReadbackRT(*view_)
+                        : RenderTargetHandle();
+        filament::RenderTarget* native_gs_rt = nullptr;
+        if (gs_rt) {
+            auto weak_rt = resource_mgr->GetRenderTarget(gs_rt);
+            if (auto rt_sptr = weak_rt.lock()) {
+                native_gs_rt = rt_sptr.get();
+            }
+        }
+
+        if (!depth_image_ && run_gs_pipeline && native_view_rt) {
+            // Issue both readPixels (base + GS overlay) together, then do one
+            // more flushAndWait to collect both callbacks synchronously.
+            //
+            // Metal readPixels from a render target only supports RGBA+UBYTE,
+            // not RGB+UBYTE (Metal has no native RGB texture format).  Always
+            // read RGBA8 for the base and strip alpha when n_channels_==3.
+            // On GL, RGBA also works fine — use one path for both backends.
+
+            const size_t n_pixels = static_cast<size_t>(width_) * height_;
+
+            // Scratch buffers for the two parallel readPixels callbacks.
+            std::vector<uint8_t> base_rgba(n_pixels * 4);
+            std::vector<float> gs_f32;
+
+            PixelBufferDescriptor base_pd(
+                    base_rgba.data(), base_rgba.size(), PixelDataFormat::RGBA,
+                    PixelDataType::UBYTE, [](void*, size_t, void*) {}, nullptr);
+            renderer_->readPixels(native_view_rt, vp.left, vp.bottom, vp.width,
+                                  vp.height, std::move(base_pd));
+
+            if (native_gs_rt) {
+                gs_f32.resize(n_pixels * 4);
+                PixelBufferDescriptor gs_pd(
+                        gs_f32.data(), gs_f32.size() * sizeof(float),
+                        PixelDataFormat::RGBA, PixelDataType::FLOAT,
+                        [](void*, size_t, void*) {}, nullptr);
+                renderer_->readPixels(native_gs_rt, vp.left, vp.bottom,
+                                      vp.width, vp.height, std::move(gs_pd));
+            }
+
+            // One more flush ensures both callbacks complete before we proceed.
+            engine_.flushAndWait();
+
+            // Unpack RGBA8 base → output buffer (strip alpha for RGB).
+            const uint8_t* src = base_rgba.data();
+            uint8_t* dst = buffer_;
+            const int nc = static_cast<int>(n_channels_);
+            const int np = static_cast<int>(n_pixels);
+            for (int i = 0; i < np; ++i) {
+                dst[i * nc + 0] = src[i * 4 + 0];
+                dst[i * nc + 1] = src[i * 4 + 1];
+                dst[i * nc + 2] = src[i * 4 + 2];
+                if (nc == 4) dst[i * nc + 3] = src[i * 4 + 3];
+            }
+            if (native_gs_rt && !gs_f32.empty()) {
+                BlendPremultipliedSplatOverRgb8(buffer_, nc, gs_f32.data(),
+                                                int(width_), int(height_));
+            }
+
+            // Deliver result now; the BeginFrame flushAndWait is a no-op since
+            // all GPU work has already been collected above.
+            if (callback_) {
+                callback_({static_cast<std::size_t>(width_),
+                           static_cast<std::size_t>(height_),
+                           static_cast<std::size_t>(n_channels_), buffer_,
+                           buffer_size_});
+                callback_ = nullptr;
+            }
+            frame_done_ = true;
+        } else if (depth_image_ && run_gs_pipeline &&
+                   gaussian_splat_renderer_) {
+            // GPU-merged depth path: the composite pass has already merged
+            // GS and Filament depth into a normalised R16UI texture.
+            // Read it back directly — no CPU merge required.
+            std::vector<std::uint16_t> merged_u16;
+            const bool got_merged =
+                    gaussian_splat_renderer_->ReadMergedDepthToUint16Cpu(
+                            *view_, merged_u16,
+                            static_cast<std::uint32_t>(width_),
+                            static_cast<std::uint32_t>(height_)) &&
+                    merged_u16.size() == width_ * height_;
+            if (got_merged) {
+                // Convert normalised uint16 [0,65535] -> Filament inverse
+                // depth [0,1]. Renderer::RenderToDepthImage applies the final
+                // user-facing conversion for z_in_view_space/normalized modes.
+                float* dst = reinterpret_cast<float*>(buffer_);
+                for (size_t i = 0; i < merged_u16.size(); ++i) {
+                    dst[i] = merged_u16[i] / 65535.f;
+                }
+                if (callback_) {
+                    callback_({static_cast<std::size_t>(width_),
+                               static_cast<std::size_t>(height_), 1u, buffer_,
+                               buffer_size_});
+                    callback_ = nullptr;
+                }
+                frame_done_ = true;
+            } else {
+                // Try GS-only composite depth (R32F) when no scene depth
+                // was available for merging.
+                std::vector<float> gs_depth;
+                const bool got_gs_depth =
+                        gaussian_splat_renderer_->ReadCompositeDepthToFloatCpu(
+                                *view_, gs_depth,
+                                static_cast<std::uint32_t>(width_),
+                                static_cast<std::uint32_t>(height_)) &&
+                        gs_depth.size() == width_ * height_;
+                if (got_gs_depth) {
+                    float* dst = reinterpret_cast<float*>(buffer_);
+                    std::copy(gs_depth.begin(), gs_depth.end(), dst);
+                    if (callback_) {
+                        callback_({static_cast<std::size_t>(width_),
+                                   static_cast<std::size_t>(height_), 1u,
+                                   buffer_, buffer_size_});
+                        callback_ = nullptr;
+                    }
+                    frame_done_ = true;
+                } else {
+                    // Final fallback: Filament depth only via readPixels
+                    // (backend unsupported or no GS depth available).
+                    auto* user_param = new PBDParams(this, callback_);
+                    PixelBufferDescriptor pd(buffer_, buffer_size_,
+                                             PixelDataFormat::DEPTH_COMPONENT,
+                                             PixelDataType::FLOAT,
+                                             ReadPixelsCallback, user_param);
+                    renderer_->readPixels(vp.left, vp.bottom, vp.width,
+                                          vp.height, std::move(pd));
+                }
+            }
+        } else {
+            if (!depth_image_ && run_gs_pipeline && !native_view_rt) {
+                utility::LogWarning(
+                        "Gaussian splat offscreen: FilamentView has no render "
+                        "target; expected EnableViewCaching. Reading the "
+                        "swapchain — splat composite may be missing.");
+            }
+            auto format = (n_channels_ == 3 ? PixelDataFormat::RGB
+                                            : PixelDataFormat::RGBA);
+            auto type = PixelDataType::UBYTE;
+            if (depth_image_) {
+                format = PixelDataFormat::DEPTH_COMPONENT;
+                type = PixelDataType::FLOAT;
+            }
+            auto* user_param = new PBDParams(this, callback_);
+            PixelBufferDescriptor pd(buffer_, buffer_size_, format, type,
+                                     ReadPixelsCallback, user_param);
+            renderer_->readPixels(vp.left, vp.bottom, vp.width, vp.height,
+                                  std::move(pd));
+        }
     }
     scene_->HideRefractedMaterials(false);
 

cpp/open3d/visualization/rendering/gaussian_splat/shaders/gaussian_depth_merge.comp

@@ -0,0 +1,72 @@
+// ----------------------------------------------------------------------------
+// -                        Open3D: www.open3d.org                            -
+// ----------------------------------------------------------------------------
+// Copyright (c) 2018-2024 www.open3d.org
+// SPDX-License-Identifier: MIT
+// ----------------------------------------------------------------------------
+
+// Gaussian splat depth-merge pass.
+//
+// Merges the GS composite inverse depth (R32F, gs.composite_depth) with
+// Filament's reversed-Z scene depth to produce a single normalised R16UI depth
+// map suitable for CPU readback via glGetTexImage / GL_RED_INTEGER /
+// GL_UNSIGNED_SHORT.
+//
+// Output: nearest depth from either source, normalised to [0, 65535] where
+//   65535 → near-plane depth, 0 → far-plane depth (Filament convention).
+//
+// The pass is optional and is only dispatched when merged_depth_u16_tex is
+// allocated (i.e. an offscreen RenderToDepthImage is in progress and scene
+// depth is available). When no scene depth is present the GS composite depth
+// is read back directly and this shader is skipped.
+//
+#version 460 core
+
+layout(local_size_x = 16, local_size_y = 16) in;
+
+// View-params UBO: same layout as every other GS compute shader.
+// Provides near/far planes and viewport size.
+layout(std140, binding = 0) uniform GaussianViewParams {
+    mat4 world_from_model;
+    mat4 view_from_world;
+    mat4 clip_from_view;
+    vec4 camera_position_and_near;
+    vec4 viewport_origin_and_size;
+    uvec4 scene;
+    uvec4 tiles;
+    uvec4 limits;
+    // depth_range_and_flags: x=near, y=far, z=reserved, w=reserved
+    vec4 depth_range_and_flags;
+}
+view_params;
+
+// GS composite depth (Filament inverse convention: near=1, far=0) written by
+// gaussian_composite.comp via Linear01ToInverse().  Sampled via texelFetch.
+// Binding 15: must stay in 0..15 for Metal (texture/sampler index limit); UBO
+// remains at 0 and does not share the sampler binding space.
+layout(binding = 15) uniform sampler2D gs_depth;
+
+// Output: merged nearest depth, normalised uint16 [0, 65535].
+layout(binding = 1, r16ui) uniform writeonly uimage2D merged_depth;
+
+// Filament reversed-Z scene depth.  Binding 14 matches gaussian_composite.comp
+// so the same descriptor set works for both shaders.
+layout(binding = 14) uniform sampler2D scene_depth;
+
+void main() {
+    ivec2 pixel = ivec2(gl_GlobalInvocationID.xy);
+    ivec2 vp = ivec2(view_params.viewport_origin_and_size.zw);
+    if (pixel.x >= vp.x || pixel.y >= vp.y) return;
+
+    // Both depth inputs are in Filament reversed/inverse depth convention.
+    float gs_inv = texelFetch(gs_depth, pixel, 0).r;
+    float sc_inv = texelFetch(scene_depth, pixel, 0).r;
+
+    // In inverse depth, larger means nearer.
+    float merged = max(gs_inv, sc_inv);
+
+    // Normalise to uint16 preserving Filament convention:
+    // near -> 65535, far -> 0.
+    uint u16 = uint(clamp(merged, 0.0, 1.0) * 65535.0 + 0.5);
+    imageStore(merged_depth, pixel, uvec4(u16, 0u, 0u, 0u));
+}