From e1cc94a5f9d20e30c0afadac8bdbffd1ecdff438 Mon Sep 17 00:00:00 2001 From: Branimir Karadzic Date: Wed, 10 Jun 2026 16:01:21 -0700 Subject: [PATCH 1/4] NativeEngine: load single-file .dds/.ktx/.ktx2 cubemaps + spherical harmonics loadCubeTexture now accepts a single self-contained cubemap container (all six faces + mips), decoded via bimg::imageParse, and uploads sides 0-5 x mips. ComputeCubeSphericalPolynomial derives the diffuse-IBL spherical harmonics from the top-mip faces (port of CubeMapToSphericalPolynomialTools) and returns the polynomial coefficients to JS. This is done natively because the WebGL upload and cube-readback paths are unimplemented on native and .dds stores no SH. Re-enables 6 prefiltered-environment PBR validation tests. Pairs with BabylonJS/Babylon.js#18560 (native createCubeTexture dispatch for single-URL containers). Depends on a babylonjs dependency bump including it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Apps/Playground/Scripts/config.json | 12 - Plugins/NativeEngine/Source/NativeEngine.cpp | 274 +++++++++++++++++++ 2 files changed, 274 insertions(+), 12 deletions(-) diff --git a/Apps/Playground/Scripts/config.json b/Apps/Playground/Scripts/config.json index 3b7389558..e0ceea2e2 100644 --- a/Apps/Playground/Scripts/config.json +++ b/Apps/Playground/Scripts/config.json @@ -858,8 +858,6 @@ { "title": "NMEGLTF", "playgroundId": "#WGZLGJ#10320", - "excludeFromAutomaticTesting": true, - "reason": "Pixel comparison fails (more than 20% pixels differ)", "referenceImage": "nmegltf.png" }, { @@ -1056,15 +1054,11 @@ { "title": "Anisotropic", "playgroundId": "#MAXCNU#1", - "excludeFromAutomaticTesting": true, - "reason": "Pixel comparison fails (more than 20% pixels differ)", "referenceImage": "anisotropic.png" }, { "title": "Clear Coat", "playgroundId": "#YACNQS#2", - "excludeFromAutomaticTesting": true, - "reason": "Pixel comparison fails (more than 20% pixels differ)", "referenceImage": "clearCoat.png" }, { @@ -1571,22 +1565,16 @@ { "title": "PBRMetallicRoughnessMaterial", "playgroundId": "#2FDQT5#13", - "excludeFromAutomaticTesting": true, - "reason": "Pixel comparison fails (more than 20% pixels differ)", "referenceImage": "PBRMetallicRoughnessMaterial.png" }, { "title": "PBRSpecularGlossinessMaterial", "playgroundId": "#Z1VL3V#4", - "excludeFromAutomaticTesting": true, - "reason": "Pixel comparison fails (more than 20% pixels differ)", "referenceImage": "PBRSpecularGlossinessMaterial.png" }, { "title": "PBR", "playgroundId": "#LCA0Q4#27", - "excludeFromAutomaticTesting": true, - "reason": "Pixel comparison fails (more than 20% pixels differ)", "referenceImage": "pbr.png" }, { diff --git a/Plugins/NativeEngine/Source/NativeEngine.cpp b/Plugins/NativeEngine/Source/NativeEngine.cpp index 864031710..f3b2f6faf 100644 --- a/Plugins/NativeEngine/Source/NativeEngine.cpp +++ b/Plugins/NativeEngine/Source/NativeEngine.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #ifdef BABYLON_NATIVE_NATIVEENGINE_TEST_HOOKS #include @@ -417,6 +418,241 @@ namespace Babylon } } } + // Parse a single self-contained cubemap container (e.g. .dds / .ktx / + // .ktx2) that already holds all six faces and their mip chain. bimg + // decodes these natively, so there is no need to split into six images + // on the JS side. Unlike ParseImage (which targets single-face 2D images + // and asserts !m_cubeMap), this keeps the container as-is. + bimg::ImageContainer* ParseCubeImage(bx::AllocatorI& allocator, gsl::span data) + { + bx::ErrorIgnore parseError; + bimg::ImageContainer* image{bimg::imageParse(&allocator, data.data(), static_cast(data.size()), bimg::TextureFormat::Count, &parseError)}; + if (image == nullptr) + { + throw std::runtime_error{"Failed to parse cube image."}; + } + + if (!image->m_cubeMap) + { + bimg::imageFree(image); + throw std::runtime_error{"Image is not a cubemap."}; + } + + return image; + } + + // Port of Babylon.js CubeMapToSphericalPolynomialTools.ConvertCubeMapToSphericalPolynomial. + // Prefiltered .dds environments need diffuse-IBL spherical harmonics, which Babylon's WebGL + // path computes on the CPU from the top-mip faces. The native engine cannot read cube faces + // back from the GPU (_readTexturePixels throws for cube faces), so we compute the harmonics + // here from the bimg-decoded top mip. Returns the 9x3 polynomial coefficients in + // SphericalPolynomial.FromArray order: x, y, z, xx, yy, zz, yz, zx, xy. + std::array ComputeCubeSphericalPolynomial(bx::AllocatorI& allocator, bimg::ImageContainer* image) + { + std::array result{}; + + bimg::ImageContainer* f32{bimg::imageConvert(&allocator, bimg::TextureFormat::RGBA32F, *image, false)}; + if (f32 == nullptr) + { + return result; + } + + const uint32_t size{f32->m_width}; + constexpr double pi{3.14159265358979323846}; + + // Face orientations matching Babylon's _FileFaces, indexed by bimg cube side order + // (+X, -X, +Y, -Y, +Z, -Z): worldAxisForNormal, worldAxisForFileX, worldAxisForFileY. + struct FaceAxes + { + double n[3]; + double fx[3]; + double fy[3]; + }; + static const FaceAxes faces[6] = { + {{1, 0, 0}, {0, 0, -1}, {0, -1, 0}}, // +X right + {{-1, 0, 0}, {0, 0, 1}, {0, -1, 0}}, // -X left + {{0, 1, 0}, {1, 0, 0}, {0, 0, 1}}, // +Y up + {{0, -1, 0}, {1, 0, 0}, {0, 0, -1}}, // -Y down + {{0, 0, 1}, {1, 0, 0}, {0, -1, 0}}, // +Z front + {{0, 0, -1}, {-1, 0, 0}, {0, -1, 0}}, // -Z back + }; + + const double shConst[9] = { + std::sqrt(1.0 / (4.0 * pi)), + -std::sqrt(3.0 / (4.0 * pi)), + std::sqrt(3.0 / (4.0 * pi)), + -std::sqrt(3.0 / (4.0 * pi)), + std::sqrt(15.0 / (4.0 * pi)), + -std::sqrt(15.0 / (4.0 * pi)), + std::sqrt(5.0 / (16.0 * pi)), + -std::sqrt(15.0 / (4.0 * pi)), + std::sqrt(15.0 / (16.0 * pi)), + }; + const double cosKernel[9] = {pi, 2.0 * pi / 3.0, 2.0 * pi / 3.0, 2.0 * pi / 3.0, pi / 4.0, pi / 4.0, pi / 4.0, pi / 4.0, pi / 4.0}; + + const auto areaElement = [](double x, double y) { return std::atan2(x * y, std::sqrt(x * x + y * y + 1.0)); }; + + double sh[9][3] = {}; + double totalSolidAngle{0.0}; + + const double du{2.0 / static_cast(size)}; + const double halfTexel{0.5 * du}; + const double minUV{halfTexel - 1.0}; + const double maxHdri{4096.0}; + + for (uint16_t side = 0; side < 6; ++side) + { + bimg::ImageMip mip{}; + if (!bimg::imageGetRawData(*f32, side, 0, f32->m_data, f32->m_size, mip)) + { + continue; + } + + const float* data{reinterpret_cast(mip.m_data)}; + const FaceAxes& f{faces[side]}; + + double v{minUV}; + for (uint32_t y = 0; y < size; ++y) + { + double u{minUV}; + for (uint32_t x = 0; x < size; ++x) + { + double dir[3] = { + f.fx[0] * u + f.fy[0] * v + f.n[0], + f.fx[1] * u + f.fy[1] * v + f.n[1], + f.fx[2] * u + f.fy[2] * v + f.n[2], + }; + const double len{std::sqrt(dir[0] * dir[0] + dir[1] * dir[1] + dir[2] * dir[2])}; + dir[0] /= len; + dir[1] /= len; + dir[2] /= len; + + const double deltaSolidAngle{ + areaElement(u - halfTexel, v - halfTexel) - + areaElement(u - halfTexel, v + halfTexel) - + areaElement(u + halfTexel, v - halfTexel) + + areaElement(u + halfTexel, v + halfTexel)}; + + const size_t idx{(static_cast(y) * size + x) * 4}; + double rgb[3] = {data[idx + 0], data[idx + 1], data[idx + 2]}; + for (int c = 0; c < 3; ++c) + { + if (std::isnan(rgb[c])) + { + rgb[c] = 0.0; + } + rgb[c] = rgb[c] < 0.0 ? 0.0 : (rgb[c] > maxHdri ? maxHdri : rgb[c]); + } + + const double trig[9] = { + 1.0, + dir[1], + dir[2], + dir[0], + dir[0] * dir[1], + dir[1] * dir[2], + 3.0 * dir[2] * dir[2] - 1.0, + dir[0] * dir[2], + dir[0] * dir[0] - dir[1] * dir[1], + }; + for (int lm = 0; lm < 9; ++lm) + { + const double basis{shConst[lm] * trig[lm] * deltaSolidAngle}; + sh[lm][0] += rgb[0] * basis; + sh[lm][1] += rgb[1] * basis; + sh[lm][2] += rgb[2] * basis; + } + totalSolidAngle += deltaSolidAngle; + u += du; + } + v += du; + } + } + + bimg::imageFree(f32); + + if (totalSolidAngle <= 0.0) + { + return result; + } + + // scaleInPlace(correction) + convertIncidentRadianceToIrradiance + convertIrradianceToLambertianRadiance. + const double correction{(4.0 * pi) / totalSolidAngle}; + for (int lm = 0; lm < 9; ++lm) + { + const double scale{correction * cosKernel[lm] / pi}; + sh[lm][0] *= scale; + sh[lm][1] *= scale; + sh[lm][2] *= scale; + } + + // SphericalPolynomial.FromHarmonics (updateFromHarmonics then *1/pi). + for (int c = 0; c < 3; ++c) + { + const double l00{sh[0][c]}, l1_1{sh[1][c]}, l10{sh[2][c]}, l11{sh[3][c]}; + const double l2_2{sh[4][c]}, l2_1{sh[5][c]}, l20{sh[6][c]}, l21{sh[7][c]}, l22{sh[8][c]}; + const double invPi{1.0 / pi}; + result[0 * 3 + c] = static_cast(-1.02333 * l11 * invPi); // x + result[1 * 3 + c] = static_cast(-1.02333 * l1_1 * invPi); // y + result[2 * 3 + c] = static_cast(1.02333 * l10 * invPi); // z + result[3 * 3 + c] = static_cast((0.886277 * l00 - 0.247708 * l20 + 0.429043 * l22) * invPi); // xx + result[4 * 3 + c] = static_cast((0.886277 * l00 - 0.247708 * l20 - 0.429043 * l22) * invPi); // yy + result[5 * 3 + c] = static_cast((0.886277 * l00 + 0.495417 * l20) * invPi); // zz + result[6 * 3 + c] = static_cast(-0.858086 * l2_1 * invPi); // yz + result[7 * 3 + c] = static_cast(-0.858086 * l21 * invPi); // zx + result[8 * 3 + c] = static_cast(0.858086 * l2_2 * invPi); // xy + } + + return result; + } + + void LoadCubeTextureFromContainer(Graphics::Texture* texture, bimg::ImageContainer* image, bool srgb) + { + assert(image->m_cubeMap); + assert(image->m_width == image->m_height); + const uint32_t size{image->m_width}; + + if (texture->IsValid()) + { + if (texture->Width() != size || texture->Height() != size) + { + bimg::imageFree(image); + throw std::runtime_error{"Cannot update texture from image of different size"}; + } + } + else + { + const bool hasMips{image->m_numMips > 1}; + const bgfx::TextureFormat::Enum format{Cast(image->m_format)}; + const uint64_t flags{srgb ? BGFX_TEXTURE_SRGB : BGFX_TEXTURE_NONE}; + texture->CreateCube(static_cast(size), hasMips, 1, format, flags); + } + + // Every (side, mip) view points into the single container's backing + // store, so the allocation is released exactly once, after bgfx has + // consumed the final upload. + const uint8_t numMips{static_cast(image->m_numMips)}; + for (uint8_t side = 0; side < 6; ++side) + { + for (uint8_t mip = 0; mip < numMips; ++mip) + { + bimg::ImageMip imageMip{}; + if (bimg::imageGetRawData(*image, side, mip, image->m_data, image->m_size, imageMip)) + { + bgfx::ReleaseFn releaseFn{}; + if (side == 5 && mip == numMips - 1) + { + releaseFn = [](void*, void* userData) { + bimg::imageFree(static_cast(userData)); + }; + } + + const bgfx::Memory* mem{bgfx::makeRef(imageMip.m_data, imageMip.m_size, releaseFn, image)}; + texture->UpdateCube(0, side, mip, 0, 0, static_cast(imageMip.m_width), static_cast(imageMip.m_height), mem); + } + } + } + } #endif // BABYLON_NATIVE_PLUGIN_NATIVEENGINE_LOAD_IMAGES auto RenderTargetSamplesToBgfxMsaaFlag(uint32_t renderTargetSamples) @@ -1533,6 +1769,44 @@ namespace Babylon const auto onSuccess{info[5].As()}; const auto onError{info[6].As()}; + // A single buffer means a self-contained cubemap container (.dds / .ktx / + // .ktx2) that already holds all six faces and their mip chain; hand it to + // bimg directly instead of expecting six pre-split face images. + if (data.Length() == 1) + { + const auto typedArray{data[0u].As()}; + const auto dataSpan{gsl::make_span(static_cast(typedArray.ArrayBuffer().Data()) + typedArray.ByteOffset(), typedArray.ByteLength())}; + auto dataRef{Napi::Persistent(typedArray)}; + arcana::make_task(arcana::threadpool_scheduler, *m_cancellationSource, [dataSpan]() { + return ParseCubeImage(Graphics::DeviceContext::GetDefaultAllocator(), dataSpan); + }) + .then(arcana::inline_scheduler, *m_cancellationSource, [texture, srgb, cancellationSource{m_cancellationSource}](bimg::ImageContainer* image) { + // Compute the spherical harmonics from the decoded top mip before the upload + // hands the container's memory to bgfx. + auto sphericalPolynomial = ComputeCubeSphericalPolynomial(Graphics::DeviceContext::GetDefaultAllocator(), image); + LoadCubeTextureFromContainer(texture, image, srgb); + return sphericalPolynomial; + }) + .then(m_runtimeScheduler, *m_cancellationSource, [dataRef{std::move(dataRef)}, onSuccessRef{Napi::Persistent(onSuccess)}, onErrorRef{Napi::Persistent(onError)}, cancellationSource{m_cancellationSource}](arcana::expected, std::exception_ptr> result) { + if (result.has_error()) + { + onErrorRef.Call({}); + } + else + { + const auto& sphericalPolynomial{result.value()}; + auto array{Napi::Float32Array::New(onSuccessRef.Env(), sphericalPolynomial.size())}; + float* dst{array.Data()}; + for (size_t i = 0; i < sphericalPolynomial.size(); ++i) + { + dst[i] = sphericalPolynomial[i]; + } + onSuccessRef.Call({array}); + } + }); + return; + } + std::array, 6> dataRefs; std::array, 6> tasks; for (uint32_t face = 0; face < data.Length(); face++) From 7182ad214efacbb9d5aed16eca7139aae88c4d4c Mon Sep 17 00:00:00 2001 From: Branimir Karadzic Date: Wed, 10 Jun 2026 10:41:18 -0700 Subject: [PATCH 2/4] Re-enable blur-cube-with-the-effect-renderer validation test This EffectRenderer fullscreen-pass test rendered all-black on Native because the native engine did not honor depth-test toggles made through engine.depthCullingState.depthTest (used by EffectRenderer). Fixed upstream in the native engine (BabylonJS/Babylon.js#18558); re-enable the test here. Depends on a babylonjs dependency bump that includes the fix. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Apps/Playground/Scripts/config.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/Apps/Playground/Scripts/config.json b/Apps/Playground/Scripts/config.json index e0ceea2e2..624c116dc 100644 --- a/Apps/Playground/Scripts/config.json +++ b/Apps/Playground/Scripts/config.json @@ -2563,8 +2563,6 @@ "title": "blur-cube-with-the-effect-renderer", "playgroundId": "#4C900K#2", "renderCount": 20, - "excludeFromAutomaticTesting": true, - "reason": "Pixel comparison fails (more than 20% pixels differ)", "referenceImage": "blur-cube-with-the-effect-renderer.png" }, { From 181930cb4131ef84d263574f5a361ddd7918bc37 Mon Sep 17 00:00:00 2001 From: Branimir Karadzic Date: Fri, 12 Jun 2026 15:51:17 -0700 Subject: [PATCH 3/4] Address review: track async cube-load task + validate cube container faces/mips - loadCubeTexture single-buffer path: capture TrackAsyncTask() into the inline continuation that uploads to bgfx, so NativeEngine::Dispose() drains it before teardown frees the graphics resources it touches (matches the multi-buffer path). - LoadCubeTextureFromContainer: validate every (side, mip) is present before uploading and fail fast (freeing the container) if not, so the single release callback attached to the last upload always fires - no leak or partially initialized texture on a truncated container. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Plugins/NativeEngine/Source/NativeEngine.cpp | 42 ++++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/Plugins/NativeEngine/Source/NativeEngine.cpp b/Plugins/NativeEngine/Source/NativeEngine.cpp index f3b2f6faf..6712c1749 100644 --- a/Plugins/NativeEngine/Source/NativeEngine.cpp +++ b/Plugins/NativeEngine/Source/NativeEngine.cpp @@ -628,28 +628,44 @@ namespace Babylon texture->CreateCube(static_cast(size), hasMips, 1, format, flags); } + // The single release callback is attached to the last (side, mip) upload, so every + // expected face/mip must be present; otherwise the container would leak (the callback + // would never fire) and the texture would be left partially initialized. Validate the + // whole face/mip grid up front and fail fast. + const uint8_t numMips{static_cast(image->m_numMips)}; + for (uint8_t side = 0; side < 6; ++side) + { + for (uint8_t mip = 0; mip < numMips; ++mip) + { + bimg::ImageMip imageMip{}; + if (!bimg::imageGetRawData(*image, side, mip, image->m_data, image->m_size, imageMip)) + { + bimg::imageFree(image); + throw std::runtime_error{"Cubemap container is missing one or more faces/mips."}; + } + } + } + // Every (side, mip) view points into the single container's backing // store, so the allocation is released exactly once, after bgfx has // consumed the final upload. - const uint8_t numMips{static_cast(image->m_numMips)}; for (uint8_t side = 0; side < 6; ++side) { for (uint8_t mip = 0; mip < numMips; ++mip) { bimg::ImageMip imageMip{}; - if (bimg::imageGetRawData(*image, side, mip, image->m_data, image->m_size, imageMip)) - { - bgfx::ReleaseFn releaseFn{}; - if (side == 5 && mip == numMips - 1) - { - releaseFn = [](void*, void* userData) { - bimg::imageFree(static_cast(userData)); - }; - } + bimg::imageGetRawData(*image, side, mip, image->m_data, image->m_size, imageMip); - const bgfx::Memory* mem{bgfx::makeRef(imageMip.m_data, imageMip.m_size, releaseFn, image)}; - texture->UpdateCube(0, side, mip, 0, 0, static_cast(imageMip.m_width), static_cast(imageMip.m_height), mem); + bgfx::ReleaseFn releaseFn{}; + if (side == 5 && mip == numMips - 1) + { + releaseFn = [](void*, void* userData) { + bimg::imageFree(static_cast(userData)); + }; } + + const bgfx::Memory* mem{bgfx::makeRef(imageMip.m_data, imageMip.m_size, releaseFn, image)}; + texture->UpdateCube(0, side, mip, 0, 0, static_cast(imageMip.m_width), static_cast(imageMip.m_height), mem); } } } @@ -1780,7 +1796,7 @@ namespace Babylon arcana::make_task(arcana::threadpool_scheduler, *m_cancellationSource, [dataSpan]() { return ParseCubeImage(Graphics::DeviceContext::GetDefaultAllocator(), dataSpan); }) - .then(arcana::inline_scheduler, *m_cancellationSource, [texture, srgb, cancellationSource{m_cancellationSource}](bimg::ImageContainer* image) { + .then(arcana::inline_scheduler, *m_cancellationSource, [texture, srgb, cancellationSource{m_cancellationSource}, asyncTaskScope{TrackAsyncTask()}](bimg::ImageContainer* image) { // Compute the spherical harmonics from the decoded top mip before the upload // hands the container's memory to bgfx. auto sphericalPolynomial = ComputeCubeSphericalPolynomial(Graphics::DeviceContext::GetDefaultAllocator(), image); From ea0d1ebda900835ad1a6694d95cdf0e230927e11 Mon Sep 17 00:00:00 2001 From: Branimir Karadzic Date: Fri, 12 Jun 2026 16:30:43 -0700 Subject: [PATCH 4/4] NativeEngine: cache numMips in cube-container loops, matching sibling upload loops Use the same or (uint8_t mip = 0, numMips = image->m_numMips; ...) idiom as LoadTextureFromImage / LoadCubeTextureFromImages instead of a standalone cached local, for consistency. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Plugins/NativeEngine/Source/NativeEngine.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Plugins/NativeEngine/Source/NativeEngine.cpp b/Plugins/NativeEngine/Source/NativeEngine.cpp index 6712c1749..72b04c0fe 100644 --- a/Plugins/NativeEngine/Source/NativeEngine.cpp +++ b/Plugins/NativeEngine/Source/NativeEngine.cpp @@ -632,10 +632,9 @@ namespace Babylon // expected face/mip must be present; otherwise the container would leak (the callback // would never fire) and the texture would be left partially initialized. Validate the // whole face/mip grid up front and fail fast. - const uint8_t numMips{static_cast(image->m_numMips)}; for (uint8_t side = 0; side < 6; ++side) { - for (uint8_t mip = 0; mip < numMips; ++mip) + for (uint8_t mip = 0, numMips = image->m_numMips; mip < numMips; ++mip) { bimg::ImageMip imageMip{}; if (!bimg::imageGetRawData(*image, side, mip, image->m_data, image->m_size, imageMip)) @@ -651,13 +650,13 @@ namespace Babylon // consumed the final upload. for (uint8_t side = 0; side < 6; ++side) { - for (uint8_t mip = 0; mip < numMips; ++mip) + for (uint8_t mip = 0, numMips = image->m_numMips; mip < numMips; ++mip) { bimg::ImageMip imageMip{}; bimg::imageGetRawData(*image, side, mip, image->m_data, image->m_size, imageMip); bgfx::ReleaseFn releaseFn{}; - if (side == 5 && mip == numMips - 1) + if (side == 5 && mip == image->m_numMips - 1) { releaseFn = [](void*, void* userData) { bimg::imageFree(static_cast(userData));