fgviewer: visualize intermediate buffers (#9862)

* Intermediate Buffer Visualization: Enabled live monitoring of internal
  `FrameGraph` render targets directly in the `fgviewer` web UI.
* HTTP Polling Architecture: Switched from WebSocket binary pushes to native
  `<img>` polling (`/api/image`)
* Robust Resource Tracking: Replaced string-based lookups with
  `(ViewId, ResourceId)` composite keys to prevent cross-view collisions and
  ensure accurate reads.
* Format Post-Processing: Extracted readback conversions (HDR tonemapping,
  depth normalization, MSAA downsampling, single-channel expansion) into
  `DebugServer`.
* UI Polish: Added a live-updating full-screen image modal and explicitly
  filtered internal debug passes from the Graphviz/JSON exports to prevent
  DOM thrashing.
* WIP: Currently Unsupported:
  * GL backend: failed with:
      OpenGL framebuffer error 0x8cd6 (GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT) in "readTexture" at line 4198
  * Mipmaps & Subresources: Reading specific mip levels or array layers is
    explicitly skipped.
  * Shadowmaps: Variance Shadow Maps (VSM) will physically evaluate to `0.0`
    and appear completely empty/black in scenes without active shadow
    casters (due to inverted-Z).
  * Stencil: Resolving and reading stencil buffer data is not supported by
    the backend.
* WIP: Untested outside of MacOS+metal/vk

Author: poweifeng

filament/CMakeLists.txt

@@ -156,6 +156,7 @@ set(SRCS
         src/ds/StructureDescriptorSet.cpp
         src/fg/Blackboard.cpp
         src/fg/DependencyGraph.cpp
+        src/fg/FgviewerManager.cpp
         src/fg/FrameGraph.cpp
         src/fg/FrameGraphPass.cpp
         src/fg/FrameGraphResources.cpp

filament/backend/src/DataReshaper.h

@@ -284,6 +284,7 @@ class DataReshaper {
                 switch (srcType) {
                     case UBYTE: reshaper = reshapeImageImpl<float, uint8_t>; break;
                     case FLOAT: reshaper = reshapeImageImpl<float, float>; break;
+                    case HALF: reshaper = reshapeImageImpl<float, math::half>; break;
                     case INT: reshaper = reshapeImageImpl<float, int32_t>; break;
                     case UINT: reshaper = reshapeImageImpl<float, uint32_t>; break;
                     case UINT_10F_11F_11F_REV:
@@ -300,6 +301,7 @@ class DataReshaper {
                 switch (srcType) {
                     case UBYTE: reshaper = reshapeImageImpl<int32_t, uint8_t>; break;
                     case FLOAT: reshaper = reshapeImageImpl<int32_t, float>; break;
+                    case HALF: reshaper = reshapeImageImpl<int32_t, math::half>; break;
                     case INT: reshaper = reshapeImageImpl<int32_t, int32_t>; break;
                     case UINT: reshaper = reshapeImageImpl<int32_t, uint32_t>; break;
                     case UINT_10F_11F_11F_REV:
@@ -316,6 +318,7 @@ class DataReshaper {
                 switch (srcType) {
                     case UBYTE: reshaper = reshapeImageImpl<uint32_t, uint8_t>; break;
                     case FLOAT: reshaper = reshapeImageImpl<uint32_t, float>; break;
+                    case HALF: reshaper = reshapeImageImpl<uint32_t, math::half>; break;
                     case INT: reshaper = reshapeImageImpl<uint32_t, int32_t>; break;
                     case UINT: reshaper = reshapeImageImpl<uint32_t, uint32_t>; break;
                     case UINT_10F_11F_11F_REV:

filament/src/PostProcessManager.cpp

@@ -3687,7 +3687,7 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::blitDepth(FrameGraph& fg,
 
 FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
         utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> const input,
-        FrameGraphTexture::Descriptor outDesc) noexcept {
+        FrameGraphTexture::Descriptor outDesc, utils::CString customPassName) noexcept {
 
     // Don't do anything if we're not a MSAA buffer
     auto const& inDesc = fg.getDescriptor(input);
@@ -3701,7 +3701,8 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
     //     through shaders or some other manipulation.
     if ((isDepthFormat(inDesc.format) || isStencilFormat(inDesc.format)) &&
             (!mDepthStencilResolveSupported)) {
-        return resolveDepthWithShader(fg, outputBufferName, input, outDesc);
+        return resolveDepthWithShader(fg, outputBufferName, input, outDesc,
+                std::move(customPassName));
     }
 
     outDesc.width = inDesc.width;
@@ -3714,7 +3715,8 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
         FrameGraphId<FrameGraphTexture> output;
     };
 
-    auto const& ppResolve = fg.addPass<ResolveData>("resolve",
+    auto const& ppResolve = fg.addPass<ResolveData>(
+            customPassName.empty() ? "resolve" : customPassName.c_str(),
             [&](FrameGraph::Builder& builder, auto& data) {
                 data.input = builder.read(input, FrameGraphTexture::Usage::BLIT_SRC);
                 data.output = builder.createTexture(outputBufferName, outDesc);
@@ -3742,7 +3744,7 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
 
 FrameGraphId<FrameGraphTexture> PostProcessManager::resolveDepthWithShader(FrameGraph& fg,
         utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> const input,
-        FrameGraphTexture::Descriptor outDesc) noexcept {
+        FrameGraphTexture::Descriptor outDesc, utils::CString customPassName) noexcept {
 
     // Don't do anything if we're not a MSAA buffer
     auto const& inDesc = fg.getDescriptor(input);
@@ -3765,7 +3767,8 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolveDepthWithShader(Frame
         FrameGraphId<FrameGraphTexture> output;
     };
 
-    auto const& ppResolve = fg.addPass<ResolveData>("resolveDepthWithShader",
+    auto const& ppResolve = fg.addPass<ResolveData>(
+            customPassName.empty() ? "resolveDepthWithShader" : customPassName.c_str(),
             [&](FrameGraph::Builder& builder, auto& data) {
                 data.input = builder.sample(input);
                 data.output = builder.createTexture(outputBufferName, outDesc);

filament/src/PostProcessManager.h

@@ -302,16 +302,19 @@ class PostProcessManager {
 
     // Resolves base level of input and outputs a texture from outDesc.
     // outDesc with, height, format and samples will be overridden.
-    FrameGraphId<FrameGraphTexture> resolve(FrameGraph& fg,
-            utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> input,
-            FrameGraphTexture::Descriptor outDesc) noexcept;
+    // customPassName is an optional parameter to override the default "resolve" pass name.
+    FrameGraphId<FrameGraphTexture> resolve(FrameGraph& fg, utils::StaticString outputBufferName,
+            FrameGraphId<FrameGraphTexture> input, FrameGraphTexture::Descriptor outDesc,
+            utils::CString customPassName = {}) noexcept;
 
     // Resolves base level of input and outputs a texture from outDesc using a shader instead of
     // driver-implemented API.
     // outDesc with, height, format and samples will be overridden.
+    // customPassName is an optional parameter to override the default "resolveDepthWithShader" pass
+    // name.
     FrameGraphId<FrameGraphTexture> resolveDepthWithShader(FrameGraph& fg,
             utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> input,
-            FrameGraphTexture::Descriptor outDesc) noexcept;
+            FrameGraphTexture::Descriptor outDesc, utils::CString customPassName = {}) noexcept;
 
     FrameGraphId<FrameGraphTexture> gaussianBlurPass(FrameGraph& fg,
             FrameGraphId<FrameGraphTexture> input,

filament/src/details/Engine.cpp

@@ -96,6 +96,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#if FILAMENT_ENABLE_FGVIEWER
+#include "fg/FgviewerManager.h"
+#endif
+
 #include "generated/resources/materials.h"
 
 using namespace filament::math;
@@ -108,6 +112,27 @@ using namespace filaflat;
 
 namespace {
 
+#if FILAMENT_ENABLE_FGVIEWER || FILAMENT_ENABLE_MATDBG
+utils::CString getPortString(std::string_view serviceType) {
+    #ifndef __ANDROID__
+    char const* portString = getenv(serviceType.data());
+    #else
+    char const* portString = [&]() -> char const*{
+        if (serviceType == "FILAMENT_MATDBG_PORT") {
+            return "8081";
+        } else if (serviceType == "FILAMENT_FGVIEWER_PORT") {
+            return "8085";
+        }
+        return nullptr;
+    }();
+    #endif
+    if (portString) {
+        return utils::CString { portString };
+    }
+    return {};
+}
+#endif // FILAMENT_ENABLE_FGVIEWER || FILAMENT_ENABLE_MATDBG
+
 Platform::DriverConfig getDriverConfig(FEngine* instance) {
     Platform::DriverConfig const driverConfig {
         .featureFlagManager = instance,
@@ -870,13 +895,8 @@ int FEngine::loop() {
     }
 
 #if FILAMENT_ENABLE_MATDBG
-    #ifdef __ANDROID__
-        const char* portString = "8081";
-    #else
-        const char* portString = getenv("FILAMENT_MATDBG_PORT");
-    #endif
-    if (portString != nullptr) {
-        const int port = atoi(portString);
+    if (auto portString = getPortString("FILAMENT_MATDBG_PORT"); !portString.empty()) {
+        const int port = atoi(portString.c_str());
 
         ShaderLanguage preferredLanguage = ShaderLanguage::UNSPECIFIED;
         if (mBackend == Backend::METAL) {
@@ -900,20 +920,13 @@ int FEngine::loop() {
 #endif
 
 #if FILAMENT_ENABLE_FGVIEWER // NOLINT(*-include-cleaner)
-#ifdef __ANDROID__
-    const char* fgviewerPortString = "8085";
-#else
-    const char* fgviewerPortString = getenv("FILAMENT_FGVIEWER_PORT");
-#endif
-    if (fgviewerPortString != nullptr) {
-        const int fgviewerPort = atoi(fgviewerPortString);
-        debug.fgviewerServer = new fgviewer::DebugServer(fgviewerPort);
-
+    if (auto portString = getPortString("FILAMENT_FGVIEWER_PORT"); !portString.empty()) {
+        debug.fgviewer = new FgviewerManager(*this, std::move(portString));
         // Sometimes the server can fail to spin up (e.g. if the above port is already in use).
         // When this occurs, carry onward, developers can look at civetweb.txt for details.
-        if (!debug.fgviewerServer->isReady()) {
-            delete debug.fgviewerServer;
-            debug.fgviewerServer = nullptr;
+        if (!debug.fgviewer->isReady()) {
+            delete debug.fgviewer;
+            debug.fgviewer = nullptr;
         }
     }
 #endif
@@ -930,8 +943,8 @@ int FEngine::loop() {
     }
 #endif
 #if FILAMENT_ENABLE_FGVIEWER
-    if(debug.fgviewerServer) {
-        delete debug.fgviewerServer;
+    if (debug.fgviewer) {
+        delete debug.fgviewer;
     }
 #endif
 
@@ -1723,7 +1736,7 @@ FixedCapacityVector<Variant> FEngine::getMaterialCompileVariants(
     if (Variant::isValidDepthVariant(depthVariant)) {
         // if we have a valid depth variant, add the stereo and skinning bits
         depthVariant.setStereo(view->hasStereo());
-        
+
         size_t const depthStart = variants.size();
         variants.push_back(depthVariant);
         apply(depthStart, skinning, &Variant::setSkinning);

filament/src/details/Engine.h

@@ -47,7 +47,6 @@
 #include "details/Skybox.h"
 #include "details/Sync.h"
 
-
 #include <private/filament/EngineEnums.h>
 
 #include <private/backend/CommandBufferQueue.h>
@@ -99,16 +98,8 @@ using MaterialKey = uint32_t;
 } // namespace filament::matdbg
 #endif
 
-#if FILAMENT_ENABLE_FGVIEWER
-#include <fgviewer/DebugServer.h>
-#else
-namespace filament::fgviewer {
-    class DebugServer;
-} // namespace filament::fgviewer
-#endif
-
 namespace filament {
-
+class FgviewerManager;
 class Renderer;
 class MaterialParser;
 class TextureCacheDisposer;
@@ -771,7 +762,7 @@ class FEngine : public Engine, public utils::FeatureFlagManager {
             bool combine_multiview_images = false;
         } stereo;
         matdbg::DebugServer* server = nullptr;
-        fgviewer::DebugServer* fgviewerServer = nullptr;
+        FgviewerManager* fgviewer = nullptr;
     } debug;
 };
 

filament/src/details/Renderer.cpp

@@ -884,6 +884,13 @@ void FRenderer::renderJob(DriverApi& driver, RootArenaScope& rootArenaScope, FVi
      */
     FrameGraph fg(*mResourceAllocator,
         isProtectedContent ? FrameGraph::Mode::PROTECTED : FrameGraph::Mode::UNPROTECTED);
+
+#if FILAMENT_ENABLE_FGVIEWER
+    if (UTILS_LIKELY(engine.debug.fgviewer)) {
+        fg.setFgviewerData(engine.debug.fgviewer, &view);
+    }
+#endif
+
     auto& blackboard = fg.getBlackboard();
 
     PostProcessManager::ScreenSpaceRefConfig const ssrConfig = PostProcessManager::prepareMipmapSSR(
@@ -1550,18 +1557,10 @@ void FRenderer::renderJob(DriverApi& driver, RootArenaScope& rootArenaScope, FVi
 //    fg.forwardResource(fgViewRenderTarget, debug ? debug : input);
 
     fg.forwardResource(fgViewRenderTarget, input);
-
     fg.present(fgViewRenderTarget);
 
     fg.compile();
 
-#if FILAMENT_ENABLE_FGVIEWER
-    fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer;
-    if (UTILS_LIKELY(fgviewerServer)) {
-        fgviewerServer->update(view.getViewHandle(), fg.getFrameGraphInfo(view.getName()));
-    }
-#endif
-
     //utils::io::sstream graphviz;
     //fg.export_graphviz(graphviz, view.getName());
     //DLOG(INFO) << graphviz.c_str();

filament/src/details/View.cpp

@@ -84,6 +84,10 @@
 
 using namespace utils;
 
+#if FILAMENT_ENABLE_FGVIEWER
+#include "fg/FgviewerManager.h"
+#endif
+
 namespace filament {
 
 using namespace backend;
@@ -172,10 +176,10 @@ FView::FView(FEngine& engine)
 #endif
 
 #if FILAMENT_ENABLE_FGVIEWER
-    fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer;
-    if (UTILS_LIKELY(fgviewerServer)) {
+    FgviewerManager* fgviewerManager = engine.debug.fgviewer;
+    if (UTILS_LIKELY(fgviewerManager)) {
         mFrameGraphViewerViewHandle =
-            fgviewerServer->createView(utils::CString(getName()));
+            fgviewerManager->createView(utils::CString(getName()));
     }
 #endif
 
@@ -225,9 +229,9 @@ void FView::terminate(FEngine& engine) {
 #endif
 
 #if FILAMENT_ENABLE_FGVIEWER
-    fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer;
-    if (UTILS_LIKELY(fgviewerServer)) {
-        fgviewerServer->destroyView(mFrameGraphViewerViewHandle);
+    FgviewerManager* fgviewerManager = engine.debug.fgviewer;
+    if (UTILS_LIKELY(fgviewerManager)) {
+        fgviewerManager->destroyView(mFrameGraphViewerViewHandle);
     }
 #endif
 }