From 496a1efd91da3df8d55ac921118c0d83c120a6aa Mon Sep 17 00:00:00 2001 From: Powei Feng Date: Mon, 30 Mar 2026 11:25:27 -0700 Subject: [PATCH] fgviewer: visualize intermediate buffers * Intermediate Buffer Visualization: Enabled live monitoring of internal `FrameGraph` render targets directly in the `fgviewer` web UI. * HTTP Polling Architecture: Switched from WebSocket binary pushes to native `` polling (`/api/image`) * Robust Resource Tracking: Replaced string-based lookups with `(ViewId, ResourceId)` composite keys to prevent cross-view collisions and ensure accurate reads. * Format Post-Processing: Extracted readback conversions (HDR tonemapping, depth normalization, MSAA downsampling, single-channel expansion) into `DebugServer`. * UI Polish: Added a live-updating full-screen image modal and explicitly filtered internal debug passes from the Graphviz/JSON exports to prevent DOM thrashing. * WIP: Currently Unsupported: * GL backend: failed with: OpenGL framebuffer error 0x8cd6 (GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT) in "readTexture" at line 4198 * Mipmaps & Subresources: Reading specific mip levels or array layers is explicitly skipped. * Shadowmaps: Variance Shadow Maps (VSM) will physically evaluate to `0.0` and appear completely empty/black in scenes without active shadow casters (due to inverted-Z). * Stencil: Resolving and reading stencil buffer data is not supported by the backend. * WIP: Untested outside of MacOS+metal/vk --- filament/CMakeLists.txt | 1 + filament/backend/src/DataReshaper.h | 3 + filament/src/PostProcessManager.cpp | 13 +- filament/src/PostProcessManager.h | 11 +- filament/src/details/Engine.cpp | 57 ++-- filament/src/details/Engine.h | 13 +- filament/src/details/Renderer.cpp | 15 +- filament/src/details/View.cpp | 16 +- filament/src/fg/DependencyGraph.cpp | 73 ++++- filament/src/fg/FgviewerManager.cpp | 280 +++++++++++++++++ filament/src/fg/FgviewerManager.h | 87 ++++++ filament/src/fg/FrameGraph.cpp | 95 +++++- filament/src/fg/FrameGraph.h | 34 +++ libs/fgviewer/CMakeLists.txt | 3 +- libs/fgviewer/include/fgviewer/DebugServer.h | 84 +++++- .../include/fgviewer/FrameGraphInfo.h | 3 + libs/fgviewer/src/ApiHandler.cpp | 200 +++++++++++-- libs/fgviewer/src/ApiHandler.h | 8 +- libs/fgviewer/src/DebugServer.cpp | 259 +++++++++++++++- libs/fgviewer/src/FrameGraphInfo.cpp | 2 +- libs/fgviewer/src/JsonWriter.cpp | 5 + libs/fgviewer/src/WebSocketHandler.cpp | 59 ---- libs/fgviewer/src/WebSocketHandler.h | 49 --- libs/fgviewer/web/api.js | 24 +- libs/fgviewer/web/app.js | 281 ++++++++++++++++-- 25 files changed, 1436 insertions(+), 239 deletions(-) create mode 100644 filament/src/fg/FgviewerManager.cpp create mode 100644 filament/src/fg/FgviewerManager.h delete mode 100644 libs/fgviewer/src/WebSocketHandler.cpp delete mode 100644 libs/fgviewer/src/WebSocketHandler.h diff --git a/filament/CMakeLists.txt b/filament/CMakeLists.txt index 33c6c0de13db..b18f9754f3cc 100644 --- a/filament/CMakeLists.txt +++ b/filament/CMakeLists.txt @@ -156,6 +156,7 @@ set(SRCS src/ds/StructureDescriptorSet.cpp src/fg/Blackboard.cpp src/fg/DependencyGraph.cpp + src/fg/FgviewerManager.cpp src/fg/FrameGraph.cpp src/fg/FrameGraphPass.cpp src/fg/FrameGraphResources.cpp diff --git a/filament/backend/src/DataReshaper.h b/filament/backend/src/DataReshaper.h index 0e182ecbfe07..474659fb41b9 100644 --- a/filament/backend/src/DataReshaper.h +++ b/filament/backend/src/DataReshaper.h @@ -284,6 +284,7 @@ class DataReshaper { switch (srcType) { case UBYTE: reshaper = reshapeImageImpl; break; case FLOAT: reshaper = reshapeImageImpl; break; + case HALF: reshaper = reshapeImageImpl; break; case INT: reshaper = reshapeImageImpl; break; case UINT: reshaper = reshapeImageImpl; break; case UINT_10F_11F_11F_REV: @@ -300,6 +301,7 @@ class DataReshaper { switch (srcType) { case UBYTE: reshaper = reshapeImageImpl; break; case FLOAT: reshaper = reshapeImageImpl; break; + case HALF: reshaper = reshapeImageImpl; break; case INT: reshaper = reshapeImageImpl; break; case UINT: reshaper = reshapeImageImpl; break; case UINT_10F_11F_11F_REV: @@ -316,6 +318,7 @@ class DataReshaper { switch (srcType) { case UBYTE: reshaper = reshapeImageImpl; break; case FLOAT: reshaper = reshapeImageImpl; break; + case HALF: reshaper = reshapeImageImpl; break; case INT: reshaper = reshapeImageImpl; break; case UINT: reshaper = reshapeImageImpl; break; case UINT_10F_11F_11F_REV: diff --git a/filament/src/PostProcessManager.cpp b/filament/src/PostProcessManager.cpp index 2067a64177a3..3440ccf59f7d 100644 --- a/filament/src/PostProcessManager.cpp +++ b/filament/src/PostProcessManager.cpp @@ -3687,7 +3687,7 @@ FrameGraphId PostProcessManager::blitDepth(FrameGraph& fg, FrameGraphId PostProcessManager::resolve(FrameGraph& fg, utils::StaticString outputBufferName, FrameGraphId const input, - FrameGraphTexture::Descriptor outDesc) noexcept { + FrameGraphTexture::Descriptor outDesc, utils::CString customPassName) noexcept { // Don't do anything if we're not a MSAA buffer auto const& inDesc = fg.getDescriptor(input); @@ -3701,7 +3701,8 @@ FrameGraphId PostProcessManager::resolve(FrameGraph& fg, // through shaders or some other manipulation. if ((isDepthFormat(inDesc.format) || isStencilFormat(inDesc.format)) && (!mDepthStencilResolveSupported)) { - return resolveDepthWithShader(fg, outputBufferName, input, outDesc); + return resolveDepthWithShader(fg, outputBufferName, input, outDesc, + std::move(customPassName)); } outDesc.width = inDesc.width; @@ -3714,7 +3715,8 @@ FrameGraphId PostProcessManager::resolve(FrameGraph& fg, FrameGraphId output; }; - auto const& ppResolve = fg.addPass("resolve", + auto const& ppResolve = fg.addPass( + customPassName.empty() ? "resolve" : customPassName.c_str(), [&](FrameGraph::Builder& builder, auto& data) { data.input = builder.read(input, FrameGraphTexture::Usage::BLIT_SRC); data.output = builder.createTexture(outputBufferName, outDesc); @@ -3742,7 +3744,7 @@ FrameGraphId PostProcessManager::resolve(FrameGraph& fg, FrameGraphId PostProcessManager::resolveDepthWithShader(FrameGraph& fg, utils::StaticString outputBufferName, FrameGraphId const input, - FrameGraphTexture::Descriptor outDesc) noexcept { + FrameGraphTexture::Descriptor outDesc, utils::CString customPassName) noexcept { // Don't do anything if we're not a MSAA buffer auto const& inDesc = fg.getDescriptor(input); @@ -3765,7 +3767,8 @@ FrameGraphId PostProcessManager::resolveDepthWithShader(Frame FrameGraphId output; }; - auto const& ppResolve = fg.addPass("resolveDepthWithShader", + auto const& ppResolve = fg.addPass( + customPassName.empty() ? "resolveDepthWithShader" : customPassName.c_str(), [&](FrameGraph::Builder& builder, auto& data) { data.input = builder.sample(input); data.output = builder.createTexture(outputBufferName, outDesc); diff --git a/filament/src/PostProcessManager.h b/filament/src/PostProcessManager.h index bc0eb64d29f4..135808368622 100644 --- a/filament/src/PostProcessManager.h +++ b/filament/src/PostProcessManager.h @@ -302,16 +302,19 @@ class PostProcessManager { // Resolves base level of input and outputs a texture from outDesc. // outDesc with, height, format and samples will be overridden. - FrameGraphId resolve(FrameGraph& fg, - utils::StaticString outputBufferName, FrameGraphId input, - FrameGraphTexture::Descriptor outDesc) noexcept; + // customPassName is an optional parameter to override the default "resolve" pass name. + FrameGraphId resolve(FrameGraph& fg, utils::StaticString outputBufferName, + FrameGraphId input, FrameGraphTexture::Descriptor outDesc, + utils::CString customPassName = {}) noexcept; // Resolves base level of input and outputs a texture from outDesc using a shader instead of // driver-implemented API. // outDesc with, height, format and samples will be overridden. + // customPassName is an optional parameter to override the default "resolveDepthWithShader" pass + // name. FrameGraphId resolveDepthWithShader(FrameGraph& fg, utils::StaticString outputBufferName, FrameGraphId input, - FrameGraphTexture::Descriptor outDesc) noexcept; + FrameGraphTexture::Descriptor outDesc, utils::CString customPassName = {}) noexcept; FrameGraphId gaussianBlurPass(FrameGraph& fg, FrameGraphId input, diff --git a/filament/src/details/Engine.cpp b/filament/src/details/Engine.cpp index 4b2cf1cbd8ac..61ba957a823b 100644 --- a/filament/src/details/Engine.cpp +++ b/filament/src/details/Engine.cpp @@ -96,6 +96,10 @@ #include #include +#if FILAMENT_ENABLE_FGVIEWER +#include "fg/FgviewerManager.h" +#endif + #include "generated/resources/materials.h" using namespace filament::math; @@ -108,6 +112,27 @@ using namespace filaflat; namespace { +#if FILAMENT_ENABLE_FGVIEWER || FILAMENT_ENABLE_MATDBG +utils::CString getPortString(std::string_view serviceType) { + #ifndef __ANDROID__ + char const* portString = getenv(serviceType.data()); + #else + char const* portString = [&]() -> char const*{ + if (serviceType == "FILAMENT_MATDBG_PORT") { + return "8081"; + } else if (serviceType == "FILAMENT_FGVIEWER_PORT") { + return "8085"; + } + return nullptr; + }(); + #endif + if (portString) { + return utils::CString { portString }; + } + return {}; +} +#endif // FILAMENT_ENABLE_FGVIEWER || FILAMENT_ENABLE_MATDBG + Platform::DriverConfig getDriverConfig(FEngine* instance) { Platform::DriverConfig const driverConfig { .featureFlagManager = instance, @@ -885,13 +910,8 @@ int FEngine::loop() { } #if FILAMENT_ENABLE_MATDBG - #ifdef __ANDROID__ - const char* portString = "8081"; - #else - const char* portString = getenv("FILAMENT_MATDBG_PORT"); - #endif - if (portString != nullptr) { - const int port = atoi(portString); + if (auto portString = getPortString("FILAMENT_MATDBG_PORT"); !portString.empty()) { + const int port = atoi(portString.c_str()); ShaderLanguage preferredLanguage = ShaderLanguage::UNSPECIFIED; if (mBackend == Backend::METAL) { @@ -915,20 +935,13 @@ int FEngine::loop() { #endif #if FILAMENT_ENABLE_FGVIEWER // NOLINT(*-include-cleaner) -#ifdef __ANDROID__ - const char* fgviewerPortString = "8085"; -#else - const char* fgviewerPortString = getenv("FILAMENT_FGVIEWER_PORT"); -#endif - if (fgviewerPortString != nullptr) { - const int fgviewerPort = atoi(fgviewerPortString); - debug.fgviewerServer = new fgviewer::DebugServer(fgviewerPort); - + if (auto portString = getPortString("FILAMENT_FGVIEWER_PORT"); !portString.empty()) { + debug.fgviewer = new FgviewerManager(*this, std::move(portString)); // Sometimes the server can fail to spin up (e.g. if the above port is already in use). // When this occurs, carry onward, developers can look at civetweb.txt for details. - if (!debug.fgviewerServer->isReady()) { - delete debug.fgviewerServer; - debug.fgviewerServer = nullptr; + if (!debug.fgviewer->isReady()) { + delete debug.fgviewer; + debug.fgviewer = nullptr; } } #endif @@ -945,8 +958,8 @@ int FEngine::loop() { } #endif #if FILAMENT_ENABLE_FGVIEWER - if(debug.fgviewerServer) { - delete debug.fgviewerServer; + if (debug.fgviewer) { + delete debug.fgviewer; } #endif @@ -1738,7 +1751,7 @@ FixedCapacityVector FEngine::getMaterialCompileVariants( if (Variant::isValidDepthVariant(depthVariant)) { // if we have a valid depth variant, add the stereo and skinning bits depthVariant.setStereo(view->hasStereo()); - + size_t const depthStart = variants.size(); variants.push_back(depthVariant); apply(depthStart, skinning, &Variant::setSkinning); diff --git a/filament/src/details/Engine.h b/filament/src/details/Engine.h index 592aba64a7b1..0514484f52cb 100644 --- a/filament/src/details/Engine.h +++ b/filament/src/details/Engine.h @@ -47,7 +47,6 @@ #include "details/Skybox.h" #include "details/Sync.h" - #include #include @@ -99,16 +98,8 @@ using MaterialKey = uint32_t; } // namespace filament::matdbg #endif -#if FILAMENT_ENABLE_FGVIEWER -#include -#else -namespace filament::fgviewer { - class DebugServer; -} // namespace filament::fgviewer -#endif - namespace filament { - +class FgviewerManager; class Renderer; class MaterialParser; class TextureCacheDisposer; @@ -771,7 +762,7 @@ class FEngine : public Engine, public utils::FeatureFlagManager { bool combine_multiview_images = false; } stereo; matdbg::DebugServer* server = nullptr; - fgviewer::DebugServer* fgviewerServer = nullptr; + FgviewerManager* fgviewer = nullptr; } debug; }; diff --git a/filament/src/details/Renderer.cpp b/filament/src/details/Renderer.cpp index 1181ddee2f40..2dbace1d109a 100644 --- a/filament/src/details/Renderer.cpp +++ b/filament/src/details/Renderer.cpp @@ -884,6 +884,13 @@ void FRenderer::renderJob(DriverApi& driver, RootArenaScope& rootArenaScope, FVi */ FrameGraph fg(*mResourceAllocator, isProtectedContent ? FrameGraph::Mode::PROTECTED : FrameGraph::Mode::UNPROTECTED); + +#if FILAMENT_ENABLE_FGVIEWER + if (UTILS_LIKELY(engine.debug.fgviewer)) { + fg.setFgviewerData(engine.debug.fgviewer, &view); + } +#endif + auto& blackboard = fg.getBlackboard(); PostProcessManager::ScreenSpaceRefConfig const ssrConfig = PostProcessManager::prepareMipmapSSR( @@ -1550,18 +1557,10 @@ void FRenderer::renderJob(DriverApi& driver, RootArenaScope& rootArenaScope, FVi // fg.forwardResource(fgViewRenderTarget, debug ? debug : input); fg.forwardResource(fgViewRenderTarget, input); - fg.present(fgViewRenderTarget); fg.compile(); -#if FILAMENT_ENABLE_FGVIEWER - fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer; - if (UTILS_LIKELY(fgviewerServer)) { - fgviewerServer->update(view.getViewHandle(), fg.getFrameGraphInfo(view.getName())); - } -#endif - //utils::io::sstream graphviz; //fg.export_graphviz(graphviz, view.getName()); //DLOG(INFO) << graphviz.c_str(); diff --git a/filament/src/details/View.cpp b/filament/src/details/View.cpp index 16e87c8efae6..8a6d2db51cf2 100644 --- a/filament/src/details/View.cpp +++ b/filament/src/details/View.cpp @@ -84,6 +84,10 @@ using namespace utils; +#if FILAMENT_ENABLE_FGVIEWER +#include "fg/FgviewerManager.h" +#endif + namespace filament { using namespace backend; @@ -172,10 +176,10 @@ FView::FView(FEngine& engine) #endif #if FILAMENT_ENABLE_FGVIEWER - fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer; - if (UTILS_LIKELY(fgviewerServer)) { + FgviewerManager* fgviewerManager = engine.debug.fgviewer; + if (UTILS_LIKELY(fgviewerManager)) { mFrameGraphViewerViewHandle = - fgviewerServer->createView(utils::CString(getName())); + fgviewerManager->createView(utils::CString(getName())); } #endif @@ -225,9 +229,9 @@ void FView::terminate(FEngine& engine) { #endif #if FILAMENT_ENABLE_FGVIEWER - fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer; - if (UTILS_LIKELY(fgviewerServer)) { - fgviewerServer->destroyView(mFrameGraphViewerViewHandle); + FgviewerManager* fgviewerManager = engine.debug.fgviewer; + if (UTILS_LIKELY(fgviewerManager)) { + fgviewerManager->destroyView(mFrameGraphViewerViewHandle); } #endif } diff --git a/filament/src/fg/DependencyGraph.cpp b/filament/src/fg/DependencyGraph.cpp index f1d85c7d514c..717682b9ffaa 100644 --- a/filament/src/fg/DependencyGraph.cpp +++ b/filament/src/fg/DependencyGraph.cpp @@ -23,6 +23,10 @@ #include #include +#if FILAMENT_ENABLE_FGVIEWER +#include +#endif + #include #include @@ -155,14 +159,33 @@ void DependencyGraph::export_graphviz(utils::io::ostream& out, char const* name) auto const& nodes = mNodes; - for (Node const* node : nodes) { + auto skipNode = [](char const* name) { +#if FILAMENT_ENABLE_FGVIEWER + if (!name) { + return false; + } + std::string_view view{ name }; + return view == std::string_view(fgviewer::READBACK_PASS_NAME) || + view == std::string_view(fgviewer::RESOLVED_MONITOR_PASS_NAME); +#else + return false; +#endif + }; + + for (Node const* node: nodes) { + if (skipNode(node->getName())) { + continue; + } uint32_t id = node->getId(); utils::CString s = node->graphvizify(); out << "\"N" << id << "\" " << s.c_str() << "\n"; } out << "\n"; - for (Node const* node : nodes) { + for (Node const* node: nodes) { + if (skipNode(node->getName())) { + continue; + } uint32_t id = node->getId(); auto edges = getOutgoingEdges(node); @@ -174,22 +197,48 @@ void DependencyGraph::export_graphviz(utils::io::ostream& out, char const* name) // render the valid edges if (first != pos) { - out << "N" << id << " -> { "; - while (first != pos) { - Node const* ref = getNode((*first++)->to); - out << "N" << ref->getId() << " "; + bool hasValidEdges = false; + for (auto it = first; it != pos; ++it) { + Node const* ref = getNode((*it)->to); + if (skipNode(ref->getName())) { + continue; + } + hasValidEdges = true; + } + if (hasValidEdges) { + out << "N" << id << " -> { "; + while (first != pos) { + Node const* ref = getNode((*first++)->to); + if (skipNode(ref->getName())) { + continue; + } + out << "N" << ref->getId() << " "; + } + out << "} [color=" << s.c_str() << "2]\n"; } - out << "} [color=" << s.c_str() << "2]\n"; } // render the invalid edges if (first != edges.end()) { - out << "N" << id << " -> { "; - while (first != edges.end()) { - Node const* ref = getNode((*first++)->to); - out << "N" << ref->getId() << " "; + bool hasInvalidEdges = false; + for (auto it = first; it != edges.end(); ++it) { + Node const* ref = getNode((*it)->to); + if (skipNode(ref->getName())) { + continue; + } + hasInvalidEdges = true; + } + if (hasInvalidEdges) { + out << "N" << id << " -> { "; + while (first != edges.end()) { + Node const* ref = getNode((*first++)->to); + if (skipNode(ref->getName())) { + continue; + } + out << "N" << ref->getId() << " "; + } + out << "} [color=" << s.c_str() << "4 style=dashed]\n"; } - out << "} [color=" << s.c_str() << "4 style=dashed]\n"; } } diff --git a/filament/src/fg/FgviewerManager.cpp b/filament/src/fg/FgviewerManager.cpp new file mode 100644 index 000000000000..7ca6193e4ae6 --- /dev/null +++ b/filament/src/fg/FgviewerManager.cpp @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if FILAMENT_ENABLE_FGVIEWER + +#include "fg/FgviewerManager.h" + +#include "details/View.h" + +#include "fg/FrameGraph.h" +#include "fg/FrameGraphId.h" +#include "fg/FrameGraphResources.h" +#include "fg/FrameGraphTexture.h" + +#include +#include + +#include +#include +#include + +#include +#include +#include + +namespace filament { + +FgviewerManager::FgviewerManager(FEngine& engine, utils::CString const& serverPort) + : mServer(std::make_unique(atoi(serverPort.c_str()), + [this](fgviewer::ViewHandle viewId, uint32_t id, utils::CString const& name, + Request::Callback&& callback) { + requestTextureReadback(viewId, id, name, std::move(callback)); + })), + mPostProcessManager(engine.getPostProcessManager()) {} + +bool FgviewerManager::isReady() const { return mServer->isReady(); } + +fgviewer::ViewHandle FgviewerManager::createView(utils::CString name) { + return mServer->createView(std::move(name)); +} + +void FgviewerManager::destroyView(fgviewer::ViewHandle h) { + mServer->destroyView(h); +} + +void FgviewerManager::addReadbacksToFramegraph(FrameGraph& fg, + backend::Handle viewTarget, uint32_t viewId) { + std::vector requests; + + { + std::unique_lock lock(mReadbackRequestsMutex); + std::swap(requests, mReadbackRequests); + } + + // Keep requests that belong to other views in the queue + std::vector remainingRequests; + + for (auto& request: requests) { + auto emptyResponse = [&request]() { + if (!request.callback) { + return; + } + request.callback(std::vector{}, 0, 0, + fgviewer::DebugServer::PixelDataFormat::RGBA, + fgviewer::DebugServer::FormatInfo{}); + }; + + if (request.viewId != viewId) { + remainingRequests.push_back(std::move(request)); + continue; + } + + FrameGraphId fgTexture = + fg.getTextureByIdName(request.id, request.name.c_str()); + if (!fgTexture.isInitialized()) { + utils::slog.e << "[fgviewer] Requested texture id " << request.id << " " + << request.name.c_str() << " not found in FrameGraph." + << "not found in FrameGraph." << utils::io::endl; + + // Dump all resources for debugging + utils::slog.e << "[fgviewer] Dumping all FrameGraph resources:" << utils::io::endl; + fg.dumpResources(); + emptyResponse(); + continue; + } + + FrameGraphTexture::Descriptor const& texDescOriginal = fg.getDescriptor(fgTexture); + utils::CString const requestName = request.name; + std::string_view const requestNameView{ requestName.data(), requestName.length() }; + bool const isViewRenderTarget = requestNameView == "viewRenderTarget"; + + // TODO: mip or multi-layer readback is not supported for fgviewer. + bool const isMip = requestNameView.find("mip") != std::string_view::npos || + requestNameView.find("Layer") != std::string_view::npos; + + if (isMip) { + utils::slog.w << "[fgviewer] WARNING: Readback for subresources (" << requestName.c_str() + << " ) is not supported." << utils::io::endl; + emptyResponse(); + continue; + } + + if (texDescOriginal.samples > 1 && !isViewRenderTarget) { + PostProcessManager& ppm = mPostProcessManager; + fgTexture = ppm.resolve(fg, "Resolved Monitor", fgTexture, { .levels = 1 }, + utils::CString(fgviewer::RESOLVED_MONITOR_PASS_NAME)); + } + + FrameGraphTexture::Descriptor const& texDesc = fg.getDescriptor(fgTexture); + bool const isDepth = + backend::isDepthFormat(texDesc.format) || backend::isStencilFormat(texDesc.format); + + struct ReadbackPassData { + FrameGraphId texture; + }; + + fg.addPass( + fgviewer::READBACK_PASS_NAME, + [&](FrameGraph::Builder& builder, ReadbackPassData& data) { + FrameGraphTexture::Usage usage = + isViewRenderTarget ? FrameGraphTexture::Usage::COLOR_ATTACHMENT + : FrameGraphTexture::Usage::SAMPLEABLE; + data.texture = builder.read(fgTexture, usage); + builder.sideEffect(); // Ensure this pass is not culled + }, + [request = std::move(request), requestName, isDepth, isViewRenderTarget, + viewTarget](FrameGraphResources const& resources, + ReadbackPassData const& data, backend::DriverApi& d) { + const FrameGraphTexture::Descriptor& desc = + resources.getDescriptor(data.texture); + + backend::TextureFormat texFormat = desc.format; + bool const isD24 = (texFormat == backend::TextureFormat::DEPTH24 || + texFormat == backend::TextureFormat::DEPTH24_STENCIL8); + bool const isD16 = (texFormat == backend::TextureFormat::DEPTH16); + bool const isD32F = (texFormat == backend::TextureFormat::DEPTH32F || + texFormat == backend::TextureFormat::DEPTH32F_STENCIL8); + + backend::PixelBufferDescriptor::PixelDataFormat format = + isDepth ? backend::PixelDataFormat::DEPTH_COMPONENT + : backend::PixelDataFormat::RGBA; + + bool const isFloat = texFormat == backend::TextureFormat::RGB16F || + texFormat == backend::TextureFormat::RGBA16F || + texFormat == backend::TextureFormat::R11F_G11F_B10F || + texFormat == backend::TextureFormat::RGB9_E5; + + bool const isR8 = texFormat == backend::TextureFormat::R8 || + texFormat == backend::TextureFormat::R8_SNORM || + texFormat == backend::TextureFormat::R8UI || + texFormat == backend::TextureFormat::R8I; + + backend::PixelBufferDescriptor::PixelDataType type; + if (isDepth) { + if (isD32F) { + type = backend::PixelDataType::FLOAT; + } else if (isD16) { + type = backend::PixelDataType::USHORT; + } else { + type = backend::PixelDataType::UINT; + } + } else { + type = isFloat ? backend::PixelDataType::FLOAT + : backend::PixelDataType::UBYTE; + } + + fgviewer::DebugServer::PixelDataFormat const targetFormat = + fgviewer::DebugServer::Format::RGBA; + + const size_t bytesPerPixel = + isDepth ? (isD32F ? 4 : (isD16 ? 2 : 4)) + : (type == backend::PixelDataType::FLOAT ? 16 : 4); + const size_t samples = desc.samples > 1 ? desc.samples : 1; + const size_t bufferSize = desc.width * desc.height * bytesPerPixel * samples; + fgviewer::DebugServer::PixelBuffer pixelBuffer(bufferSize); + void* bufferData = pixelBuffer.data(); + fgviewer::DebugServer::FormatInfo info{ + .isDepth = isDepth, + .isD16 = isD16, + .isD24 = isD24, + .isD32F = isD32F, + .isFloat = isFloat, + .isR8 = isR8, + .samples = static_cast(samples), + }; + + struct UserData { + size_t width; + size_t height; + fgviewer::DebugServer::PixelDataFormat targetFormat; + Request::Callback callback; + fgviewer::DebugServer::PixelBuffer pixelBuffer; + fgviewer::DebugServer::FormatInfo info; + }; + + backend::PixelBufferDescriptor pbd( + bufferData, bufferSize, format, type, 1, 0, 0, 0, 0, + [](void* buffer, size_t size, void* user) { + std::unique_ptr d{ static_cast(user) }; + auto& callback = d->callback; + if (callback) { + callback(std::move(d->pixelBuffer), d->width, d->height, + d->targetFormat, d->info); + } + }, + new UserData{ desc.width, desc.height, targetFormat, + std::move(request.callback), std::move(pixelBuffer), info }); + + if (isViewRenderTarget) { + if (viewTarget) { + d.readPixels(viewTarget, 0, 0, desc.width, desc.height, std::move(pbd)); + } else { + utils::slog.e << "[fgviewer] ERROR: Hardware RenderTarget is " + "uninitialized for " + << requestName.c_str() << utils::io::endl; + if (request.callback) { + request.callback(std::vector{}, 0, 0, + fgviewer::DebugServer::Format::RGBA, + fgviewer::DebugServer::FormatInfo{}); + } + } + } else { + auto hwTex = resources.getTexture(data.texture); + if (hwTex) { + auto srd = resources.getSubResourceDescriptor(data.texture); + d.readTexture(hwTex, srd.level, srd.layer, 0, 0, desc.width, + desc.height, std::move(pbd)); + } else { + utils::slog.e << "[fgviewer] ERROR: Hardware Texture is uninitialized for " + << requestName.c_str() << utils::io::endl; + if (request.callback) { + request.callback(std::vector{}, 0, 0, + fgviewer::DebugServer::Format::RGBA, + fgviewer::DebugServer::FormatInfo{}); + } + } + } + }); + } + + if (!remainingRequests.empty()) { + std::unique_lock lock(mReadbackRequestsMutex); + mReadbackRequests.insert(mReadbackRequests.end(), + std::make_move_iterator(remainingRequests.begin()), + std::make_move_iterator(remainingRequests.end())); + } +} + +void FgviewerManager::framegraphUpdated(FrameGraph& fg, FView const& view) { + auto info = fg.getFrameGraphInfo(view.getName()); + mServer->update(view.getViewHandle(), std::move(info)); +} + +void FgviewerManager::framegraphExecuted() { + mServer->tick(); +} + +void FgviewerManager::requestTextureReadback(fgviewer::ViewHandle viewId, uint32_t id, + const utils::CString& name, + Request::Callback&& callback) { + std::unique_lock lock(mReadbackRequestsMutex); + mReadbackRequests.emplace_back(Request{ viewId, id, name, std::move(callback) }); +} + +} // namespace filament + +#endif // FILAMENT_ENABLE_FGVIEWER diff --git a/filament/src/fg/FgviewerManager.h b/filament/src/fg/FgviewerManager.h new file mode 100644 index 000000000000..fab657aa2926 --- /dev/null +++ b/filament/src/fg/FgviewerManager.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_FILAMENT_FGVIEWERMANAGER_H +#define TNT_FILAMENT_FGVIEWERMANAGER_H + +#include "details/Engine.h" +#include "PostProcessManager.h" + +#include + +#include + +#include + +#include +#include +#include +#include + +namespace filament { + +class FEngine; +class FView; +class FrameGraph; + +class FgviewerManager { + +public: + FgviewerManager(FEngine& engine, utils::CString const& serverPort); + + // Encapsulates a texture readback request + struct Request { + using Callback = fgviewer::DebugServer::ReabackFinishedCallback; + fgviewer::ViewHandle viewId; + uint32_t id; + utils::CString name; + Callback callback; + }; + + // FgViewer offsers the feature of viewing intermediate rendered textures (render targets). + // Each texture client wants to view is a request to readback the corresponding texture in the + // framegraph. This method adds readback passes to the framegraph for each readback request. + void addReadbacksToFramegraph(FrameGraph& fg, + backend::Handle viewTarget, uint32_t viewId); + + // Updates FgViewer's framegraph state with the latest. + void framegraphUpdated(FrameGraph& fg, FView const& view); + + // This signals to fgviewer that the framegraph has been executed. + void framegraphExecuted(); + + // Returns whether the server has been initialized (e.g. it could have failed because another + // process owns the port). + bool isReady() const; + + fgviewer::ViewHandle createView(utils::CString name); + void destroyView(fgviewer::ViewHandle h); + +private: + void requestTextureReadback(fgviewer::ViewHandle viewId, uint32_t id, + utils::CString const& name, Request::Callback&& callback); + + std::unique_ptr mServer; + utils::Mutex mReadbackRequestsMutex; + std::vector mReadbackRequests; + + // Needed for adding a resolve pass for multi-sampled rendertargets. + PostProcessManager& mPostProcessManager; +}; + +} // namespace filament + +#endif // TNT_FILAMENT_FGVIEWERMANAGER_H diff --git a/filament/src/fg/FrameGraph.cpp b/filament/src/fg/FrameGraph.cpp index 94216eeafb9a..8d29f656701b 100644 --- a/filament/src/fg/FrameGraph.cpp +++ b/filament/src/fg/FrameGraph.cpp @@ -26,7 +26,7 @@ #include "FrameGraphRenderPass.h" #include "FrameGraphTexture.h" -#include "details/Engine.h" +#include "details/View.h" #include #include @@ -77,7 +77,9 @@ FrameGraphId FrameGraph::Builder::declareRenderPass( // ------------------------------------------------------------------------------------------------ -FrameGraph::FrameGraph(TextureCacheInterface& resourceAllocator, Mode const mode) + +FrameGraph::FrameGraph(TextureCacheInterface& resourceAllocator, + Mode const mode) : mResourceAllocator(resourceAllocator), mArena("FrameGraph Arena", 262144), mMode(mode), @@ -92,6 +94,14 @@ FrameGraph::FrameGraph(TextureCacheInterface& resourceAllocator, Mode const mode mPassNodes.reserve(64); } +#if FILAMENT_ENABLE_FGVIEWER +void FrameGraph::setFgviewerData(FgviewerManager* fgviewer, FView const* view) { + mFgviewer = fgviewer; + mView = view; +} +#endif + + UTILS_NOINLINE void FrameGraph::destroyInternal() noexcept { // the order of destruction is important here @@ -124,6 +134,12 @@ FrameGraph& FrameGraph::compile() noexcept { FILAMENT_TRACING_CALL(FILAMENT_TRACING_CATEGORY_FILAMENT); +#if FILAMENT_ENABLE_FGVIEWER + // Add passes for reading back textures + mFgviewer->addReadbacksToFramegraph(*this, mView->getRenderTargetHandle(), + mView->getViewHandle()); +#endif + DependencyGraph& dependencyGraph = mGraph; // first we cull unreachable nodes @@ -194,6 +210,9 @@ FrameGraph& FrameGraph::compile() noexcept { pNode->resolveResourceUsage(dependencyGraph); } +#if FILAMENT_ENABLE_FGVIEWER + mFgviewer->framegraphUpdated(*this, *mView); +#endif return *this; } @@ -235,6 +254,10 @@ void FrameGraph::execute(backend::DriverApi& driver) noexcept { driver.popGroupMarker(); } driver.popGroupMarker(); + +#if FILAMENT_ENABLE_FGVIEWER + mFgviewer->framegraphExecuted(); +#endif } void FrameGraph::addPresentPass(const std::function& setup) noexcept { @@ -484,8 +507,8 @@ void FrameGraph::export_graphviz(utils::io::ostream& out, char const* name) cons mGraph.export_graphviz(out, name); } -fgviewer::FrameGraphInfo FrameGraph::getFrameGraphInfo(const char *viewName) const { #if FILAMENT_ENABLE_FGVIEWER +fgviewer::FrameGraphInfo FrameGraph::getFrameGraphInfo(const char* viewName) const { fgviewer::FrameGraphInfo info{utils::CString(viewName)}; std::vector passes; @@ -495,6 +518,12 @@ fgviewer::FrameGraphInfo FrameGraph::getFrameGraphInfo(const char *viewName) con PassNode *const pass = *first; ++first; + auto const nameStrView = std::string_view(pass->getName()); + if (nameStrView == fgviewer::READBACK_PASS_NAME || + nameStrView == fgviewer::RESOLVED_MONITOR_PASS_NAME) { + continue; + } + assert_invariant(!pass->isCulled()); std::vector reads; auto const &readEdges = mGraph.getIncomingEdges(pass); @@ -528,7 +557,13 @@ fgviewer::FrameGraphInfo FrameGraph::getFrameGraphInfo(const char *viewName) con } std::unordered_map resources; - for (const auto &resourceNode: mResourceNodes) { + for (const auto& resourceNode: mResourceNodes) { + auto const nameStrView = std::string_view(resourceNode->getName()); + if (nameStrView == fgviewer::READBACK_PASS_NAME || + nameStrView == fgviewer::RESOLVED_MONITOR_PASS_NAME) { + continue; + } + const FrameGraphHandle resourceHandle = resourceNode->resourceHandle; if (resources.find(resourceHandle.index) != resources.end()) continue; @@ -578,14 +613,62 @@ fgviewer::FrameGraphInfo FrameGraph::getFrameGraphInfo(const char *viewName) con info.setGraphvizData(utils::CString(out.c_str())); return info; -#else - return fgviewer::FrameGraphInfo(); +} #endif + +#if FILAMENT_ENABLE_FGVIEWER + +FrameGraphId FrameGraph::getTextureByIdName(uint32_t id, + const char* name) const { + // 1. Try the fast path: check if the requested ID matches the expected name. + if (id < mResourceSlots.size()) { + auto const& slot = mResourceSlots[id]; + if (slot.rid > 0) { + VirtualResource const* resource = mResources[slot.rid]; + if (resource && strcmp(resource->name.c_str(), name) == 0) { + FrameGraphHandle handle; + handle.index = (uint16_t) id; + handle.version = slot.version; + return FrameGraphId(handle); + } + } + } + + // 2. Slow path fallback: graph shifts across frames, so the ID might be invalid. + // Search all resource slots for a matching name. + for (size_t i = 1; i < mResourceSlots.size(); ++i) { + auto const& slot = mResourceSlots[i]; + if (slot.rid > 0) { + VirtualResource const* resource = mResources[slot.rid]; + if (resource && strcmp(resource->name.c_str(), name) == 0) { + FrameGraphHandle handle; + handle.index = (uint16_t) i; + handle.version = slot.version; + return FrameGraphId(handle); + } + } + } + + return {}; } +void FrameGraph::dumpResources() const { + for (size_t i = 1; i < mResourceSlots.size(); ++i) { + auto const& slot = mResourceSlots[i]; + if (slot.rid > 0) { + VirtualResource const* resource = mResources[slot.rid]; + if (resource) { + fprintf(stderr, "[fgviewer] Slot %zu rid %u: \"%s\"\n", i, (uint32_t) slot.rid, + resource->name.c_str()); + } + } + } +} +#endif // ------------------------------------------------------------------------------------------------ + /* * Explicit template instantiation for FrameGraphTexture which is a known type, * to reduce compile time and code size. diff --git a/filament/src/fg/FrameGraph.h b/filament/src/fg/FrameGraph.h index c209631651c9..bc24473d4ff4 100644 --- a/filament/src/fg/FrameGraph.h +++ b/filament/src/fg/FrameGraph.h @@ -38,10 +38,14 @@ #if FILAMENT_ENABLE_FGVIEWER #include +#include "fg/FgviewerManager.h" #else namespace filament::fgviewer { class FrameGraphInfo{}; } // namespace filament::fgviewer +namespace filament { + class FgviewerManager; +} #endif namespace filament { @@ -234,6 +238,13 @@ class FrameGraph { explicit FrameGraph(TextureCacheInterface& resourceAllocator, Mode mode = Mode::UNPROTECTED); + +#if FILAMENT_ENABLE_FGVIEWER + // When fgviewer is enabled, we update the fgviewer at different points of framegraph + // life-cycles. The view is useful for associating a graph with the view it's rendering into. + void setFgviewerData(FgviewerManager* manager, FView const* view); +#endif + FrameGraph(FrameGraph const&) = delete; FrameGraph& operator=(FrameGraph const&) = delete; ~FrameGraph() noexcept; @@ -448,8 +459,26 @@ class FrameGraph { * Export a fgviewer::FrameGraphInfo for current graph. * Note that this function should be called after FrameGraph::compile(). */ +#if FILAMENT_ENABLE_FGVIEWER fgviewer::FrameGraphInfo getFrameGraphInfo(const char *viewName) const; + /** + * Retrieve a texture handle by its name. This is used for debugging/visualization tools. + * @param name Name of the resource + * @return Handle to the texture, or an uninitialized handle if not found. + */ + FrameGraphId getTextureByIdName(uint32_t id, const char* name) const; + + /** + * Dump all resources to log. + */ + void dumpResources() const; +#else + fgviewer::FrameGraphInfo getFrameGraphInfo(const char*) const { + return fgviewer::FrameGraphInfo(); + } +#endif + private: friend class FrameGraphResources; friend class PassNode; @@ -547,6 +576,11 @@ class FrameGraph { Vector mResourceNodes; Vector mPassNodes; Vector::iterator mActivePassNodesEnd; + +#if FILAMENT_ENABLE_FGVIEWER + FgviewerManager* mFgviewer = nullptr; + FView const* mView = nullptr; +#endif }; template diff --git a/libs/fgviewer/CMakeLists.txt b/libs/fgviewer/CMakeLists.txt index a32e20265ec7..db089a310e9b 100644 --- a/libs/fgviewer/CMakeLists.txt +++ b/libs/fgviewer/CMakeLists.txt @@ -24,7 +24,6 @@ set(SRCS src/DebugServer.cpp src/FrameGraphInfo.cpp src/JsonWriter.cpp - src/WebSocketHandler.cpp ) # ================================================================================================== @@ -71,7 +70,7 @@ target_link_libraries(${TARGET} PUBLIC utils ) -target_include_directories(${TARGET} PRIVATE ${filamat_SOURCE_DIR}/src) +target_include_directories(${TARGET} PRIVATE ${filamat_SOURCE_DIR}/src ${FILAMENT}/third_party/stb) target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) set_target_properties(${TARGET} PROPERTIES FOLDER Libs) diff --git a/libs/fgviewer/include/fgviewer/DebugServer.h b/libs/fgviewer/include/fgviewer/DebugServer.h index 2d4e8b3ad62f..5e46384f009e 100644 --- a/libs/fgviewer/include/fgviewer/DebugServer.h +++ b/libs/fgviewer/include/fgviewer/DebugServer.h @@ -22,6 +22,8 @@ #include #include +#include +#include #include #include @@ -30,7 +32,6 @@ class CivetServer; namespace filament::fgviewer { using ViewHandle = uint32_t; -class WebSocketHandler; /** * Server-side frame graph debugger. @@ -40,10 +41,36 @@ class WebSocketHandler; */ class DebugServer { public: + enum class PixelDataFormat : uint8_t { R, RG, RGB, RGBA, L, LA }; + struct Format { + static constexpr PixelDataFormat R = PixelDataFormat::R; + static constexpr PixelDataFormat RG = PixelDataFormat::RG; + static constexpr PixelDataFormat RGB = PixelDataFormat::RGB; + static constexpr PixelDataFormat RGBA = PixelDataFormat::RGBA; + static constexpr PixelDataFormat L = PixelDataFormat::L; + static constexpr PixelDataFormat LA = PixelDataFormat::LA; + }; + + struct FormatInfo { + bool isDepth = false; + bool isD16 = false; + bool isD24 = false; + bool isD32F = false; + bool isFloat = false; + bool isR8 = false; + uint8_t samples = 1; + }; + + using PixelBuffer = std::vector; + using ReabackFinishedCallback = + std::function; + using ReadbackRequest = std::function; + static std::string_view const kSuccessHeader; static std::string_view const kErrorHeader; - explicit DebugServer(int port); + explicit DebugServer(int port, ReadbackRequest request = {}); ~DebugServer(); /** @@ -59,7 +86,39 @@ class DebugServer { /** * Updates the information for a given view. */ - void update(ViewHandle h, FrameGraphInfo info); + void update(ViewHandle h, FrameGraphInfo&& info); + + /** + * Ticks the server to trigger monitoring requests. + */ + void tick(); + + /** + * Set a resource to be monitored. + */ + void setResourceMonitor(ViewHandle viewId, uint32_t resourceId, + utils::CString const& resourceName, bool enabled); + + /** + * Clear all resource monitors. + */ + void clearResourceMonitors(); + + struct ActiveMonitor { + ViewHandle viewId; + uint32_t resourceId; + utils::CString name; + }; + + /** + * Get currently active resource monitors. + */ + std::vector getMonitoredResources() const; + + /** + * Get the latest rendered image for a monitored resource. + */ + std::vector getImage(ViewHandle viewId, uint32_t resourceId) const; bool isReady() const { return mServer; } @@ -70,13 +129,28 @@ class DebugServer { uint32_t mViewCounter = 0; mutable utils::Mutex mViewsMutex; + struct ResourceMonitor { + ViewHandle viewId; + uint32_t resourceId; + bool enabled = false; + bool inProgress = false; + size_t readbackTick = 0; // Marks the next tick where an actual readback will be issued. + // This relaxes the amount of read back requests per-frame. + utils::CString name; + std::vector lastImage; + }; + std::unordered_map mMonitoredResources; + mutable utils::Mutex mMonitoredResourcesMutex; + + ReadbackRequest mReadbackRequest; + + size_t mTickCount = 0; + class FileRequestHandler* mFileHandler = nullptr; class ApiHandler* mApiHandler = nullptr; - class WebSocketHandler* mWebSocketHandler = nullptr; friend class FileRequestHandler; friend class ApiHandler; - friend class WebSocketHandler; }; } // namespace filament::fgviewer diff --git a/libs/fgviewer/include/fgviewer/FrameGraphInfo.h b/libs/fgviewer/include/fgviewer/FrameGraphInfo.h index a4a6908843d6..0d06c5690303 100644 --- a/libs/fgviewer/include/fgviewer/FrameGraphInfo.h +++ b/libs/fgviewer/include/fgviewer/FrameGraphInfo.h @@ -26,6 +26,9 @@ namespace filament::fgviewer { using ResourceId = uint32_t; +constexpr const char READBACK_PASS_NAME[] = "Readback Pass"; +constexpr const char RESOLVED_MONITOR_PASS_NAME[] = "Resolved Monitor"; + class FrameGraphInfo { public: explicit FrameGraphInfo(utils::CString viewName); diff --git a/libs/fgviewer/src/ApiHandler.cpp b/libs/fgviewer/src/ApiHandler.cpp index 1f9d5fc48c9b..b07d9238a0dd 100644 --- a/libs/fgviewer/src/ApiHandler.cpp +++ b/libs/fgviewer/src/ApiHandler.cpp @@ -27,11 +27,12 @@ #include +#include #include #include #include #include -#include +#include namespace filament::fgviewer { @@ -57,6 +58,14 @@ bool ApiHandler::handleGet(CivetServer* server, struct mg_connection* conn) { return handleGetStatus(conn, request); } + if (uri.find("/api/image") == 0) { + return handleGetImage(conn, request); + } + + if (uri == "/api/monitor") { + return handleGetMonitor(conn, request); + } + if (uri == "/api/framegraphs") { std::unique_lock const lock(mServer->mViewsMutex); mg_printf(conn, kSuccessHeader.data(), "application/json"); @@ -78,13 +87,20 @@ bool ApiHandler::handleGet(CivetServer* server, struct mg_connection* conn) { } if (uri == "/api/framegraph") { - const FrameGraphInfo* result = getFrameGraphInfo(request); - if (!result) { + size_t const qlength = strlen(request->query_string); + char fgid[9] = {}; + if (mg_get_var(request->query_string, qlength, "fgid", fgid, sizeof(fgid)) < 0) { return error(__LINE__, uri); } - + uint32_t const id = strtoul(fgid, nullptr, 16); + std::unique_lock const lock(mServer->mViewsMutex); + auto const it = mServer->mViews.find(id); + if (it == mServer->mViews.end()) { + return error(__LINE__, uri); + } + FrameGraphInfo const& result = it->second; JsonWriter writer; - if (!writer.writeFrameGraphInfo(*result)) { + if (!writer.writeFrameGraphInfo(result)) { return error(__LINE__, uri); } mg_printf(conn, kSuccessHeader.data(), "application/json"); @@ -95,27 +111,127 @@ bool ApiHandler::handleGet(CivetServer* server, struct mg_connection* conn) { return error(__LINE__, uri); } -void ApiHandler::updateFrameGraph(ViewHandle view_handle) { +bool ApiHandler::handlePost(CivetServer* server, struct mg_connection* conn) { + struct mg_request_info const* request = mg_get_request_info(conn); + std::string const& uri = request->local_uri; + if (uri == "/api/monitor") { + return handlePostMonitor(conn, request); + } + if (uri == "/api/monitor/clear") { + return handlePostMonitorClear(conn, request); + } + + return error(__LINE__, uri); +} + +bool ApiHandler::handlePostMonitor(struct mg_connection* conn, + struct mg_request_info const* request) { + char postData[1024]; + int const postDataLen = mg_read(conn, postData, sizeof(postData) - 1); + if (postDataLen < 0) { + LOG(ERROR) << "mg_read failed"; + return error(__LINE__, request->local_uri); + } + postData[postDataLen] = '\0'; + + // Primitive JSON parsing + std::string_view body(postData); + auto fgidPos = body.find("\"fgid\":"); + auto idPos = body.find("\"id\":"); + auto namePos = body.find("\"name\":"); + auto enabledPos = body.find("\"enabled\":"); + if (idPos == std::string_view::npos || enabledPos == std::string_view::npos) { + LOG(ERROR) << "JSON parsing failed for id/enabled. body=" << body; + return error(__LINE__, request->local_uri); + } + + uint32_t fgid = 0; + if (fgidPos != std::string_view::npos) { + auto fgidStart = body.find("\"", fgidPos + 7); + if (fgidStart != std::string_view::npos) { + fgidStart += 1; + auto fgidEnd = body.find("\"", fgidStart); + if (fgidEnd != std::string_view::npos) { + auto sv = body.substr(fgidStart, fgidEnd - fgidStart); + try { + fgid = std::stoul(std::string(sv), nullptr, 16); + } catch (...) { + } + } + } + } + + uint32_t id = 0; + auto idStart = idPos + 4; + while (idStart < body.length() && (body[idStart] == ' ' || body[idStart] == ':')) { + idStart++; + } + auto idEnd = body.find(",", idStart); + if (idEnd == std::string_view::npos) idEnd = body.find("}", idStart); + try { + auto sv = body.substr(idStart, idEnd - idStart); + auto [ptr, ec] = std::from_chars(sv.data(), sv.data() + sv.size(), id); + if (ec != std::errc()) { + LOG(ERROR) << "from_chars failed for id. sv=" << sv; + } + } catch (...) { + LOG(ERROR) << "ID extraction threw an exception"; + return error(__LINE__, request->local_uri); + } + + std::string_view name = "Unknown"; + if (namePos != std::string_view::npos) { + auto nameStart = body.find("\"", namePos + 7); + if (nameStart != std::string_view::npos) { + nameStart++; + auto nameEnd = body.find("\"", nameStart); + if (nameEnd != std::string_view::npos) { + name = body.substr(nameStart, nameEnd - nameStart); + } + } + } + + bool const enabled = body.find("true", enabledPos) != std::string_view::npos; + + LOG(INFO) << "Monitor updated: fgid=" << fgid << " id=" << id << " name=" << name + << " enabled=" << enabled; + mServer->setResourceMonitor(fgid, id, utils::CString(name.data(), name.length()), enabled); + mg_printf(conn, kSuccessHeader.data(), "application/json"); + mg_printf(conn, "{ \"status\": \"ok\" }"); + return true; +} + +bool ApiHandler::handlePostMonitorClear(struct mg_connection* conn, + struct mg_request_info const* request) { + mServer->clearResourceMonitors(); + mg_printf(conn, kSuccessHeader.data(), "application/json"); + mg_printf(conn, "{ \"status\": \"ok\" }"); + return true; +} + +bool ApiHandler::handleGetMonitor(struct mg_connection* conn, + struct mg_request_info const* request) { + auto const resources = mServer->getMonitoredResources(); + mg_printf(conn, kSuccessHeader.data(), "application/json"); + mg_printf(conn, "["); + bool first = true; + for (auto const& res : resources) { + if (!first) mg_printf(conn, ","); + mg_printf(conn, "{\"fgid\":%u, \"id\":%u, \"name\":\"%s\"}", + res.viewId, res.resourceId, res.name.c_str()); + first = false; + } + mg_printf(conn, "]"); + return true; +} + +void ApiHandler::updateFrameGraph(ViewHandle view) { std::unique_lock const lock(mStatusMutex); - snprintf(statusFrameGraphId, sizeof(statusFrameGraphId), "%8.8x", view_handle); + snprintf(statusFrameGraphId, sizeof(statusFrameGraphId), "%8.8x", view); mCurrentStatus++; mStatusCondition.notify_all(); } -const FrameGraphInfo* ApiHandler::getFrameGraphInfo(struct mg_request_info const* request) { - size_t const qlength = strlen(request->query_string); - char fgid[9] = {}; - if (mg_get_var(request->query_string, qlength, "fgid", fgid, sizeof(fgid)) < 0) { - return nullptr; - } - uint32_t const id = strtoul(fgid, nullptr, 16); - std::unique_lock const lock(mServer->mViewsMutex); - const auto it = mServer->mViews.find(id); - return it == mServer->mViews.end() - ? nullptr - : &(it->second); -} - bool ApiHandler::handleGetStatus(struct mg_connection* conn, struct mg_request_info const* request) { char const* qstr = request->query_string; @@ -142,4 +258,46 @@ bool ApiHandler::handleGetStatus(struct mg_connection* conn, return true; } +bool ApiHandler::handleGetImage(struct mg_connection* conn, struct mg_request_info const* request) { + char const* qstr = request->query_string; + if (!qstr) return false; + + uint32_t fgid = 0; + uint32_t id = 0; + + size_t const qlength = strlen(qstr); + + char fgid_str[16] = {}; + if (mg_get_var(qstr, qlength, "fgid", fgid_str, sizeof(fgid_str)) > 0) { + fgid = static_cast(std::strtoul(fgid_str, nullptr, 16)); + } + + char id_str[16] = {}; + if (mg_get_var(qstr, qlength, "id", id_str, sizeof(id_str)) > 0) { + id = static_cast(std::strtoul(id_str, nullptr, 10)); + } + + std::vector pngData = mServer->getImage(fgid, id); + if (!pngData.empty()) { + mg_printf(conn, + "HTTP/1.1 200 OK\r\n" + "Content-Type: image/png\r\n" + "Cache-Control: no-cache, no-store, must-revalidate\r\n" + "Content-Length: %zu\r\n" + "Access-Control-Allow-Origin: *\r\n" + "Connection: close\r\n" + "\r\n", + pngData.size()); + mg_write(conn, pngData.data(), pngData.size()); + } else { + // Return an empty transparent 1x1 PNG or a 404 + mg_printf(conn, "HTTP/1.1 404 Not Found\r\n" + "Content-Length: 0\r\n" + "Connection: close\r\n" + "\r\n"); + } + + return true; +} + } // filament::fgviewer diff --git a/libs/fgviewer/src/ApiHandler.h b/libs/fgviewer/src/ApiHandler.h index b7e7cf7c7030..7b7948db91fd 100644 --- a/libs/fgviewer/src/ApiHandler.h +++ b/libs/fgviewer/src/ApiHandler.h @@ -38,15 +38,19 @@ class ApiHandler : public CivetHandler { : mServer(server) {} ~ApiHandler() = default; - bool handleGet(CivetServer* server, struct mg_connection* conn); + bool handleGet(CivetServer* server, struct mg_connection* conn) override; + bool handlePost(CivetServer* server, struct mg_connection* conn) override; void updateFrameGraph(ViewHandle view_handle); private: - const FrameGraphInfo* getFrameGraphInfo(struct mg_request_info const* request); + bool handlePostMonitor(struct mg_connection* conn, struct mg_request_info const* request); + bool handlePostMonitorClear(struct mg_connection* conn, struct mg_request_info const* request); + bool handleGetMonitor(struct mg_connection* conn, struct mg_request_info const* request); bool handleGetStatus(struct mg_connection* conn, struct mg_request_info const* request); + bool handleGetImage(struct mg_connection* conn, struct mg_request_info const* request); DebugServer* mServer; diff --git a/libs/fgviewer/src/DebugServer.cpp b/libs/fgviewer/src/DebugServer.cpp index 722f3158b095..fc1eafcc05ba 100644 --- a/libs/fgviewer/src/DebugServer.cpp +++ b/libs/fgviewer/src/DebugServer.cpp @@ -18,7 +18,9 @@ #include #include "ApiHandler.h" -#include "WebSocketHandler.h" + +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include #include @@ -27,6 +29,7 @@ #include #include +#include #include #include @@ -39,6 +42,7 @@ namespace { std::string const BASE_URL = "libs/fgviewer/web"; + } // anonymous #else @@ -55,6 +59,24 @@ struct Asset { std::unordered_map ASSET_MAP; +constexpr int READBACK_BUFFER = 16; + +struct _Random { +private: + std::random_device rd; + std::mt19937 gen; + std::uniform_int_distribution distr; + +public: + _Random() + : rd(), + gen(rd()), + distr(-READBACK_BUFFER / 2, READBACK_BUFFER) {} + + int get() { return distr(gen); } +} randGen; + + } // anonymous #endif // SERVE_FROM_SOURCE_TREE @@ -62,6 +84,25 @@ std::unordered_map ASSET_MAP; namespace filament::fgviewer { using namespace utils; +using PixelDataFormat = DebugServer::PixelDataFormat; + +uint8_t getNumberOfComponents(PixelDataFormat format) { + switch (format) { + case PixelDataFormat::R: + return 1; + case PixelDataFormat::RG: + return 2; + case PixelDataFormat::RGB: + return 3; + case PixelDataFormat::RGBA: + return 4; + case PixelDataFormat::L: + return 1; + case PixelDataFormat::LA: + return 2; + } + return 0; +} std::string_view const DebugServer::kSuccessHeader = "HTTP/1.1 200 OK\r\nContent-Type: %s\r\n" @@ -105,7 +146,8 @@ class FileRequestHandler : public CivetHandler { DebugServer* mServer; }; -DebugServer::DebugServer(int port) { +DebugServer::DebugServer(int port, ReadbackRequest request) + : mReadbackRequest(std::move(request)) { #if !SERVE_FROM_SOURCE_TREE ASSET_MAP["/index.html"] = { .mime = "text/html", @@ -144,7 +186,6 @@ DebugServer::DebugServer(int port) { mFileHandler = new FileRequestHandler(this); mApiHandler = new ApiHandler(this); - mWebSocketHandler = new WebSocketHandler(this); mServer->addHandler("/api", mApiHandler); mServer->addHandler("", mFileHandler); @@ -157,7 +198,6 @@ DebugServer::~DebugServer() { delete mFileHandler; delete mApiHandler; - delete mWebSocketHandler; delete mServer; } @@ -175,7 +215,7 @@ void DebugServer::destroyView(ViewHandle h) { mViews.erase(h); } -void DebugServer::update(ViewHandle h, FrameGraphInfo info) { +void DebugServer::update(ViewHandle h, FrameGraphInfo&& info) { std::unique_lock lock(mViewsMutex); const auto it = mViews.find(h); if (it == mViews.end()) { @@ -183,13 +223,218 @@ void DebugServer::update(ViewHandle h, FrameGraphInfo info) { return; } - bool has_changed = !(it->second == info); - if (!has_changed) + bool const hasChanged = !(it->second == info); + if (!hasChanged) { return; + } mViews.erase(h); mViews.emplace(h, std::move(info)); mApiHandler->updateFrameGraph(h); } +void DebugServer::tick() { + std::unique_lock lock(mMonitoredResourcesMutex); + for (auto& [key, monitor]: mMonitoredResources) { + if (!monitor.enabled || monitor.inProgress || monitor.readbackTick > mTickCount) { + continue; + } + monitor.readbackTick = mTickCount + READBACK_BUFFER + randGen.get(); + monitor.inProgress = true; + auto name = monitor.name; + auto viewId = monitor.viewId; + auto id = monitor.resourceId; + mReadbackRequest(viewId, id, name, + [this, key, id, name](PixelBuffer buffer, uint32_t width, uint32_t height, + PixelDataFormat format, FormatInfo info) { + if (!buffer.data() || width == 0 || height == 0) { + std::unique_lock lock(mMonitoredResourcesMutex); + mMonitoredResources[key].inProgress = false; + return; + } + + size_t const pixelCount = width * height; + PixelBuffer converted(pixelCount * 4); + uint8_t* dst = converted.data(); + + if (info.isDepth) { + // Depth Buffer Normalization + // Transforms high-precision floating point or integer depth buffers + // (e.g. D16, D24, D32F) into a normalized 0-255 spectrum using an automatic + // min/max mapping. Needed to make the micro-variations in Z-depth visibly + // distinguishable in the UI. + std::vector depthFloats(pixelCount); + + if (info.isD32F) { + const float* src = reinterpret_cast(buffer.data()); + for (size_t i = 0; i < pixelCount; ++i) { + depthFloats[i] = src[i]; + } + } else if (info.isD16) { + const uint16_t* src = reinterpret_cast(buffer.data()); + for (size_t i = 0; i < pixelCount; ++i) { + depthFloats[i] = static_cast(src[i]) / 65535.0f; + } + } else { + const uint32_t* src = reinterpret_cast(buffer.data()); + for (size_t i = 0; i < pixelCount; ++i) { + uint32_t raw = src[i]; + if (info.isD24) { + uint32_t d24 = raw & 0x00FFFFFF; + depthFloats[i] = static_cast(d24) / 16777215.0f; + } else { + depthFloats[i] = static_cast(raw) / 4294967295.0f; + } + } + } + + float minVal = std::numeric_limits::max(); + float maxVal = std::numeric_limits::lowest(); + for (size_t i = 0; i < pixelCount; ++i) { + float v = depthFloats[i]; + if (v < minVal) minVal = v; + if (v > maxVal) maxVal = v; + } + + float range = maxVal - minVal; + if (range < 1e-6f) range = 1.0f; + + for (size_t i = 0; i < pixelCount; ++i) { + float val = (depthFloats[i] - minVal) / range; + val = std::pow(val, 0.5f); // simple gamma curve + uint8_t c = static_cast(std::clamp(val, 0.0f, 1.0f) * 255.0f); + dst[i * 4 + 0] = c; + dst[i * 4 + 1] = c; + dst[i * 4 + 2] = c; + dst[i * 4 + 3] = 255; + } + } else if (info.samples > 1) { + // MSAA Downsampling + // Converts multi-sampled buffers into standard 1x buffers by skipping over + // the extra samples. Needed to prevent the UI image from appearing + // stretched/distorted by incorrect pitch. + const uint8_t* src = reinterpret_cast(buffer.data()); + for (size_t i = 0; i < pixelCount; ++i) { + size_t srcIdx = i * 4 * info.samples; + dst[i * 4 + 0] = src[srcIdx + 0]; + dst[i * 4 + 1] = src[srcIdx + 1]; + dst[i * 4 + 2] = src[srcIdx + 2]; + dst[i * 4 + 3] = src[srcIdx + 3]; + } + } else { + if (info.isFloat) { + // HDR Tonemapping & Exposure + // Applies a basic curve to float color targets (RGB16F, R11G11B10F, + // RGB9E5). Needed because raw IEEE float bytes cannot be displayed + // natively, and HDR values typically exceed 1.0 intensity which + // requires mapping back into sRGB 0-255 boundaries. + const float* src = reinterpret_cast(buffer.data()); + for (size_t i = 0; i < pixelCount; ++i) { + float r = src[i * 4 + 0]; + float g = src[i * 4 + 1]; + float b = src[i * 4 + 2]; + + dst[i * 4 + 0] = static_cast( + std::pow(std::clamp(r, 0.0f, 1.0f), 1.0f / 2.2f) * 255.0f); + dst[i * 4 + 1] = static_cast( + std::pow(std::clamp(g, 0.0f, 1.0f), 1.0f / 2.2f) * 255.0f); + dst[i * 4 + 2] = static_cast( + std::pow(std::clamp(b, 0.0f, 1.0f), 1.0f / 2.2f) * 255.0f); + dst[i * 4 + 3] = 255; + } + } else if (info.isR8) { + // Single-Channel Grayscale Expand + // Copies the Red channel into Green and Blue for visualization. + // Needed because raw 8-bit monochromatic targets would otherwise be + // parsed as standard RGBA by the PNG encoder, resulting in a garbled + // red tint or alignment skew. + const uint8_t* src = buffer.data(); + for (size_t i = 0; i < pixelCount * 4; i += 4) { + uint8_t r = src[i]; + dst[i + 0] = r; + dst[i + 1] = r; + dst[i + 2] = r; + dst[i + 3] = 255; + } + } else { + // Standard 8-bit Buffer + // Force alpha to 255 (opaque) since UI should visualize the RGB + // contents even if the render pass outputs A=0 logic. + const uint8_t* src = buffer.data(); + for (size_t i = 0; i < pixelCount * 4; i += 4) { + dst[i + 0] = src[i + 0]; + dst[i + 1] = src[i + 1]; + dst[i + 2] = src[i + 2]; + dst[i + 3] = 255; + } + } + } + + int const components = getNumberOfComponents(format); + int pngSize = 0; + unsigned char* pngData = stbi_write_png_to_mem(converted.data(), + width * components, width, height, components, &pngSize); + + if (pngData) { + std::unique_lock lock(mMonitoredResourcesMutex); + mMonitoredResources[key].lastImage.assign(pngData, pngData + pngSize); + free(pngData); + } + + { + std::unique_lock lock(mMonitoredResourcesMutex); + mMonitoredResources[key].inProgress = false; + } + }); + } + + mTickCount++; +} + +std::vector DebugServer::getImage(ViewHandle viewId, uint32_t resourceId) const { + std::unique_lock lock(mMonitoredResourcesMutex); + uint64_t key = ((uint64_t) viewId << 32) | resourceId; + auto it = mMonitoredResources.find(key); + if (it != mMonitoredResources.end()) { + return it->second.lastImage; + } + return {}; +} + +void DebugServer::clearResourceMonitors() { + std::unique_lock lock(mMonitoredResourcesMutex); + for (auto& [key, monitor] : mMonitoredResources) { + monitor.enabled = false; + monitor.inProgress = false; + monitor.lastImage.clear(); + } +} + +std::vector DebugServer::getMonitoredResources() const { + std::vector out; + std::unique_lock lock(mMonitoredResourcesMutex); + for (auto const& [key, monitor] : mMonitoredResources) { + if (monitor.enabled) { + out.push_back({monitor.viewId, monitor.resourceId, monitor.name}); + } + } + return out; +} + +void DebugServer::setResourceMonitor(ViewHandle viewId, uint32_t resourceId, + utils::CString const& resourceName, bool enabled) { + LOG(INFO) << "[fgviewer] DebugServer(" << this << ")::setResourceMonitor: view=" << viewId + << " id=" << resourceId << ", " << resourceName.c_str() << " -> " + << (enabled ? "enabled" : "disabled"); + std::unique_lock lock(mMonitoredResourcesMutex); + uint64_t key = ((uint64_t) viewId << 32) | resourceId; + mMonitoredResources[key].viewId = viewId; + mMonitoredResources[key].resourceId = resourceId; + mMonitoredResources[key].enabled = enabled; + mMonitoredResources[key].name = resourceName; + if (!enabled) { + mMonitoredResources[key].inProgress = false; + } +} + } // namespace filament::fgviewer diff --git a/libs/fgviewer/src/FrameGraphInfo.cpp b/libs/fgviewer/src/FrameGraphInfo.cpp index 8abf97d24ea2..18873e5a7ab8 100644 --- a/libs/fgviewer/src/FrameGraphInfo.cpp +++ b/libs/fgviewer/src/FrameGraphInfo.cpp @@ -46,7 +46,7 @@ FrameGraphInfo::Pass::Pass(uint32_t id, utils::CString name, std::vector - -#include - -namespace filament::fgviewer { - -using namespace utils; - -WebSocketHandler::WebSocketHandler(DebugServer* server) - : mServer(server) {} - -bool WebSocketHandler::handleConnection(CivetServer* server, const struct mg_connection* conn) { - LOG(INFO) << "[fgviewer] WebSocket connected."; - return true; -} - -void WebSocketHandler::handleReadyState(CivetServer* server, struct mg_connection* conn) { - std::unique_lock lock(mMutex); - mConnections.insert(conn); -} - -bool WebSocketHandler::handleData(CivetServer* server, struct mg_connection* conn, int bits, - char* data, size_t data_len) { - // For now, we don't expect any data from the client. - return true; -} - -void WebSocketHandler::handleClose(CivetServer* server, const struct mg_connection* conn) { - LOG(INFO) << "[fgviewer] WebSocket disconnected."; - std::unique_lock lock(mMutex); - mConnections.erase(const_cast(conn)); -} - -void WebSocketHandler::broadcast(const char* data, size_t len) { - std::unique_lock lock(mMutex); - for (auto* conn: mConnections) { - mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data, len); - } -} - -} // namespace filament::fgviewer diff --git a/libs/fgviewer/src/WebSocketHandler.h b/libs/fgviewer/src/WebSocketHandler.h deleted file mode 100644 index 6e70c9ba5245..000000000000 --- a/libs/fgviewer/src/WebSocketHandler.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2026 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FGVIEWER_WEBSOCKET_HANDLER_H -#define FGVIEWER_WEBSOCKET_HANDLER_H - -#include - -#include -#include - -namespace filament::fgviewer { - -class DebugServer; - -class WebSocketHandler : public CivetWebSocketHandler { -public: - explicit WebSocketHandler(DebugServer* server); - - bool handleConnection(CivetServer* server, const struct mg_connection* conn) override; - void handleReadyState(CivetServer* server, struct mg_connection* conn) override; - bool handleData(CivetServer* server, struct mg_connection* conn, int bits, char* data, - size_t data_len) override; - void handleClose(CivetServer* server, const struct mg_connection* conn) override; - - void broadcast(const char* data, size_t len); - -private: - DebugServer const* mServer; - utils::Mutex mMutex; - std::unordered_set mConnections; -}; - -} // namespace filament::fgviewer - -#endif // FGVIEWER_WEBSOCKET_HANDLER_H diff --git a/libs/fgviewer/web/api.js b/libs/fgviewer/web/api.js index 586e1ce53be6..ce00ec070bb5 100644 --- a/libs/fgviewer/web/api.js +++ b/libs/fgviewer/web/api.js @@ -41,6 +41,26 @@ async function fetchFrameGraph(fgid) { return fgInfo; } +async function fetchMonitoredResources() { + return await _fetchJson(`api/monitor`); +} + +async function clearMonitoredResources() { + const response = await fetch(`api/monitor/clear`, { + method: "POST" + }); + return await response.json(); +} + +async function toggleMonitor(fgid, id, name, enabled) { + const response = await fetch(`api/monitor`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fgid: fgid, id: id, name: name, enabled: enabled }) + }); + return await response.json(); +} + const STATUS_LOOP_TIMEOUT = 3000; const STATUS_CONNECTED = 1; @@ -55,9 +75,9 @@ async function statusLoop(isConnected, onStatus) { const fgid = await _fetchText("api/status" + (isConnected() ? '' : '?firstTime')); // A first-time request returned successfully if (fgid === '0') { - onStatus(STATUS_CONNECTED); + await onStatus(STATUS_CONNECTED); } else if (fgid !== '1') { - onStatus(STATUS_FRAMEGRAPH_UPDATED, fgid); + await onStatus(STATUS_FRAMEGRAPH_UPDATED, fgid); } // fgid == '1' is no-op, just loop again statusLoop(isConnected, onStatus); } catch { diff --git a/libs/fgviewer/web/app.js b/libs/fgviewer/web/app.js index 6cac711fcd89..10f1cf610bab 100644 --- a/libs/fgviewer/web/app.js +++ b/libs/fgviewer/web/app.js @@ -51,6 +51,8 @@ const IS_SUBRESOURCE_KEY = 'is_subresource_of' const VIEW_MODE_TABLE = 'table'; const VIEW_MODE_GRAPHVIZ = 'graphviz'; +const MONITORED_REFRESH_RATE = 10; + const COLORS = [ "#FFB3BA", "#BAFFC9", @@ -558,6 +560,7 @@ class FrameGraphTable extends LitElement { frameGraphData: {type: Object, state: true}, // Expecting a JSON frame graph structure selectedResourceId: {type: Number, attribute: 'selected-resource'}, selectedPassIndex: {type: Number, attribute: 'selected-pass'}, + monitoredResources: {type: Array, state: true}, tooltipText: {type: String, state: true}, tooltipVisible: {type: Boolean, state: true}, tooltipX: {type: Number, state: true}, @@ -565,11 +568,44 @@ class FrameGraphTable extends LitElement { }; } + _toggleMonitor(resource) { + const enabled = !this.monitoredResources.some(r => r.id === resource.id); + toggleMonitor(this.selectedFrameGraph, resource.id, resource.name, enabled).then(() => { + // Update local state immediately upon success + if (!enabled) { + const idx = this.monitoredResources.findIndex(r => r.id === resource.id); + if (idx >= 0) { + const newResources = [...this.monitoredResources]; + newResources.splice(idx, 1); + this.monitoredResources = newResources; + + // Fire custom event to update parent state + this.dispatchEvent(new CustomEvent('update-monitored', { + detail: newResources, + bubbles: true, + composed: true + })); + } + } else { + // If enabling, we'll let the polling loop populate the image url. + // But we can add a placeholder so the checkbox immediately checks. + const newResources = [...this.monitoredResources, { id: resource.id, name: resource.name, url: '' }]; + this.monitoredResources = newResources; + + this.dispatchEvent(new CustomEvent('update-monitored', { + detail: newResources, + bubbles: true, + composed: true + })); + } + }); + } constructor() { super(); this.frameGraphData = null; this.selectedResourceId = -1; this.selectedPassIndex = -1; + this.monitoredResources = []; this.expandedResourceSet = new Set(); this.subresourceToParent = {}; this._hoverTimeout = null; @@ -764,6 +800,10 @@ class FrameGraphTable extends LitElement { ` : nothing} ${resource.name} + ${hasSubresources && !isExpanded ? html`(${subresourceIds.length})` : nothing} ${this._renderResourceUsage(allPasses, resourceIds, defaultColor, lastRow)} @@ -865,12 +905,19 @@ class GraphvizView extends LitElement { return; try { - const viz = d3.select(container) - .graphviz({ useWorker: false }) - .zoom(true) - .fit(true); + if (!this._viz) { + this._viz = d3.select(container) + .graphviz({ useWorker: false }) + .zoom(true) + .fit(true); + } - viz.renderDot(this.graphvizData); + if (this._lastGraphvizData === this.graphvizData) { + return; + } + this._lastGraphvizData = this.graphvizData; + + this._viz.renderDot(this.graphvizData); } catch (error) { console.error('Failed to render graphviz:', error); container.innerHTML = `
Failed to render graphviz: ${error.message}
`; @@ -891,6 +938,137 @@ class GraphvizView extends LitElement { customElements.define("graphviz-view", GraphvizView); +class MonitorPanel extends LitElement { + static get properties() { + return { + monitoredResources: { type: Array }, + expandedResourceId: { type: Number } + }; + } + + static get styles() { + return css` + :host { + display: flex; + flex-direction: row; + gap: 10px; + pointer-events: none; + margin-left: 20px; + margin-top: 40px; + } + + .monitor-card { + background: white; + border: 1px solid #ccc; + border-radius: 8px; + padding: 10px; + pointer-events: auto; + width: 125px; + transition: all 0.2s ease; + cursor: pointer; + } + + .monitor-card:hover { + box-shadow: 0 4px 8px rgba(0,0,0,0.3); + transform: translateY(-2px); + } + + .monitor-title { + margin-bottom: 5px; + font-size: 7px; + display: flex; + justify-content: space-between; + } + + .monitor-image { + width: 100%; + height: auto; + display: block; + border: 1px solid #eee; + } + + .modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.8); + z-index: 1000; + display: flex; + align-items: center; + justify-content: center; + pointer-events: auto; + } + + .modal-content { + background: white; + padding: 15px; + border-radius: 8px; + display: flex; + flex-direction: column; + max-width: 90vw; + max-height: 90vh; + box-shadow: 0 10px 25px rgba(0,0,0,0.5); + } + + .modal-title { + font-size: 16px; + font-weight: bold; + margin-bottom: 10px; + text-align: center; + color: #333; + } + + .modal-image { + max-width: 100%; + max-height: 80vh; + object-fit: contain; + border: 1px solid #ddd; + } + `; + } + + constructor() { + super(); + this.monitoredResources = []; + this.expandedResourceId = null; + } + + render() { + const expandedRes = this.expandedResourceId !== null + ? this.monitoredResources.find(r => r.id === this.expandedResourceId) + : null; + + // If the resource was removed from monitored list while modal is open, close it + if (this.expandedResourceId !== null && !expandedRes) { + this.expandedResourceId = null; + } + + return html` + ${this.monitoredResources.map(res => html` +
+
+ ${res.name} +
+ ${res.name} +
+ `)} + + ${expandedRes ? html` +
+
+
${expandedRes.name}
+ ${expandedRes.name} +
+
+ ` : nothing} + `; + } +} + +customElements.define("monitor-panel", MonitorPanel); + class FrameGraphViewer extends LitElement { static get styles() { return css` @@ -923,16 +1101,29 @@ class FrameGraphViewer extends LitElement { this.connected = status == STATUS_CONNECTED || status == STATUS_FRAMEGRAPH_UPDATED; if (status == STATUS_FRAMEGRAPH_UPDATED) { - let fgInfo = await fetchFrameGraph(fgid); - this.database[fgInfo.fgid] = fgInfo; - this._framegraphTable.frameGraphData = fgInfo; - this._graphvizView.graphvizData = fgInfo.graphviz; + let fgInfo = await fetchFrameGraph(fgid); + + this.database[fgid] = fgInfo; + this.requestUpdate(); + + this._framegraphTable.frameGraphData = fgInfo; + // Only update graphviz view if it's visible, else wait for tab switch + if (this.viewMode === VIEW_MODE_GRAPHVIZ) { + this._graphvizView.graphvizData = fgInfo.graphviz; + } } } ); let framegraphs = await fetchFrameGraphs(); this.database = framegraphs; + + let monitors = await fetchMonitoredResources(); + this.monitoredResources = monitors.map(m => ({ + id: m.id, + name: m.name, + url: `api/image?fgid=${m.fgid}&id=${m.id}&t=${Date.now()}` + })); } _getFrameGraph() { @@ -949,8 +1140,11 @@ class FrameGraphViewer extends LitElement { this.selectedResourceId = -1; this.selectedPassIndex = -1; this.viewMode = VIEW_MODE_TABLE; + this.monitoredResources = []; this.init(); + this._setupImagePolling(); + this.addEventListener('select-framegraph', (ev) => { this.selectedFrameGraph = ev.detail; @@ -970,8 +1164,40 @@ class FrameGraphViewer extends LitElement { ); this.addEventListener('change-view-mode', - (ev) => { + async (ev) => { this.viewMode = ev.detail; + if (this.viewMode === VIEW_MODE_GRAPHVIZ) { + await clearMonitoredResources(); + this.monitoredResources = []; + if (this.selectedFrameGraph) { + let fgInfo = await fetchFrameGraph(this.selectedFrameGraph); + this.database[this.selectedFrameGraph] = fgInfo; + this.requestUpdate(); + this._graphvizView.graphvizData = fgInfo.graphviz; + } + } else if (this.viewMode === VIEW_MODE_TABLE && this.selectedFrameGraph) { + let fgInfo = await fetchFrameGraph(this.selectedFrameGraph); + this.database[this.selectedFrameGraph] = fgInfo; + this.requestUpdate(); + this._framegraphTable.frameGraphData = fgInfo; + } + } + ); + + this.addEventListener('update-monitored', + (ev) => { + // Keep the ordering when updated from child + const newResources = ev.detail; + const fg = this._getFrameGraph(); + if (fg && fg.resources) { + const resourceOrder = Object.values(fg.resources); + newResources.sort((a, b) => { + const idxA = resourceOrder.findIndex(r => r.id === a.id); + const idxB = resourceOrder.findIndex(r => r.id === b.id); + return idxA - idxB; + }); + } + this.monitoredResources = newResources; } ); } @@ -984,6 +1210,7 @@ class FrameGraphViewer extends LitElement { selectedResourceId: {type: Number, state: true}, selectedPassIndex: {type: Number, state: true}, viewMode: {type: String, state: true}, + monitoredResources: {type: Array, state: true}, } } @@ -997,6 +1224,22 @@ class FrameGraphViewer extends LitElement { } + _setupImagePolling() { + if (this._pollingInterval) { + clearInterval(this._pollingInterval); + } + this._pollingInterval = setInterval(() => { + if (this.monitoredResources.length > 0 && this.selectedFrameGraph) { + const ts = Date.now(); + const newResources = this.monitoredResources.map(r => ({ + ...r, + url: `api/image?fgid=${this.selectedFrameGraph}&id=${r.id}&t=${ts}` + })); + this.monitoredResources = newResources; + } + }, 1000 / MONITORED_REFRESH_RATE); + } + render() { return html` - - +
+ + + +