Skip to content

Commit 3234b0b

Browse files
Merge pull request #738 from fernandotonon/feat/onnx-pbr-synth-404
feat: AI PBR map synthesis from albedo (ONNX) (#404)
2 parents ed7387d + f820169 commit 3234b0b

21 files changed

Lines changed: 2103 additions & 3 deletions

.github/workflows/deploy.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -749,6 +749,7 @@ jobs:
749749
sudo cmake -S . -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \
750750
-DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \
751751
-DENABLE_STABLE_DIFFUSION=ON \
752+
-DENABLE_ONNX=ON \
752753
-DENABLE_AUTO_UPDATER=OFF \
753754
-DASSIMP_DIR=/usr/local/lib/cmake/assimp-${{ env.ASSIMP_DIR_VERSION }} \
754755
-DASSIMP_INCLUDE_DIR=/usr/local/include/assimp \
@@ -1032,7 +1033,8 @@ jobs:
10321033
-DCMAKE_EXE_LINKER_FLAGS="--coverage" \
10331034
-DBUILD_QT_MESH_EDITOR=OFF \
10341035
-DENABLE_SENTRY=OFF \
1035-
-DENABLE_PS1_RIP=ON
1036+
-DENABLE_PS1_RIP=ON \
1037+
-DENABLE_ONNX=ON
10361038
10371039
- name: Run build-wrapper
10381040
env:
@@ -1725,6 +1727,7 @@ jobs:
17251727
-DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \
17261728
-DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \
17271729
-DENABLE_STABLE_DIFFUSION=ON \
1730+
-DENABLE_ONNX=ON \
17281731
-DCMAKE_OSX_ARCHITECTURES="$(uname -m)" \
17291732
-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \
17301733
-DASSIMP_DIR=/usr/local/lib/cmake/assimp-${{ env.ASSIMP_DIR_VERSION }} \

.gitignore

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ ui_files/
2828
ui_*
2929
*.log
3030
*.cmake
31-
!cmake/Libsodium.cmake
31+
!cmake/*.cmake
3232
*.user
3333
*.qtc_clangd*
3434
_CPack_Packages/*
@@ -97,3 +97,7 @@ docs/*
9797
minisign.key
9898
*.minisig
9999
!tests/fixtures/**/*.minisig
100+
101+
# Local ML experiment scratch (not part of the project)
102+
src/python_scripts/
103+
src/project_for_llm.py

CLAUDE.md

Lines changed: 3 additions & 0 deletions
Large diffs are not rendered by default.

CMakeLists.txt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,16 @@ if(ENABLE_STABLE_DIFFUSION)
264264
message(STATUS "AI texture generation enabled with stable-diffusion.cpp")
265265
endif()
266266
##############################################################
267+
# ONNX Runtime — AI PBR map synthesis (#404)
268+
##############################################################
269+
option(ENABLE_ONNX "Enable AI PBR map synthesis via ONNX Runtime" OFF)
270+
271+
if(ENABLE_ONNX)
272+
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/OnnxRuntime.cmake)
273+
add_definitions(-DENABLE_ONNX)
274+
message(STATUS "AI PBR map synthesis enabled with ONNX Runtime")
275+
endif()
276+
##############################################################
267277
# PS1 runtime geometry extraction (experimental)
268278
##############################################################
269279
option(ENABLE_PS1_RIP "Enable experimental PS1 runtime geometry extraction" OFF)

cmake/OnnxRuntime.cmake

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# ONNX Runtime for AI PBR map synthesis (#404).
2+
#
3+
# Downloads the official prebuilt ONNX Runtime release archive per-platform and
4+
# exposes it as an imported SHARED target `qtmesh_onnx`. Mirrors the download
5+
# pattern of cmake/Libsodium.cmake, but consumes a prebuilt binary instead of
6+
# building from source.
7+
#
8+
# macOS uses the universal2 archive (covers arm64 + x86_64) so there is no
9+
# per-arch selection trap (the lesson from libsodium being built x86_64). CoreML
10+
# execution provider ships inside that archive; the CPU EP is always present.
11+
# Windows MinGW is intentionally NOT wired here — the official Windows archive is
12+
# MSVC-built and won't link under MinGW; that path degrades gracefully (the
13+
# feature reports "rebuild with -DENABLE_ONNX"). See the #404 follow-up.
14+
15+
if(TARGET qtmesh_onnx)
16+
return()
17+
endif()
18+
19+
set(QTMESH_ONNX_VERSION "1.20.1" CACHE STRING "ONNX Runtime release version")
20+
set(QTMESH_ONNX_BASE_URL
21+
"https://github.com/microsoft/onnxruntime/releases/download/v${QTMESH_ONNX_VERSION}")
22+
23+
# Select the archive + its SHA256 for this platform.
24+
if(APPLE)
25+
set(_ort_archive "onnxruntime-osx-universal2-${QTMESH_ONNX_VERSION}.tgz")
26+
set(_ort_sha256 "da4349e01a7e997f5034563183c7183d069caadc1d95f499b560961787813efd")
27+
set(_ort_libname "libonnxruntime.${QTMESH_ONNX_VERSION}.dylib")
28+
elseif(UNIX)
29+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
30+
set(_ort_archive "onnxruntime-linux-aarch64-${QTMESH_ONNX_VERSION}.tgz")
31+
set(_ort_sha256 "ae4fedbdc8c18d688c01306b4b50c63de3445cdf2dbd720e01a2fa3810b8106a")
32+
else()
33+
set(_ort_archive "onnxruntime-linux-x64-${QTMESH_ONNX_VERSION}.tgz")
34+
set(_ort_sha256 "67db4dc1561f1e3fd42e619575c82c601ef89849afc7ea85a003abbac1a1a105")
35+
endif()
36+
set(_ort_libname "libonnxruntime.so.${QTMESH_ONNX_VERSION}")
37+
elseif(WIN32 AND NOT MINGW)
38+
set(_ort_archive "onnxruntime-win-x64-${QTMESH_ONNX_VERSION}.zip")
39+
set(_ort_sha256 "78d447051e48bd2e1e778bba378bec4ece11191c9e538cf7b2c4a4565e8f5581")
40+
set(_ort_libname "onnxruntime.dll")
41+
else()
42+
message(FATAL_ERROR "ENABLE_ONNX: unsupported platform — no ONNX Runtime archive mapping. "
43+
"Disable with -DENABLE_ONNX=OFF or add the archive here.")
44+
endif()
45+
46+
include(FetchContent)
47+
FetchContent_Declare(
48+
qtmesh_onnxruntime
49+
URL "${QTMESH_ONNX_BASE_URL}/${_ort_archive}"
50+
URL_HASH SHA256=${_ort_sha256}
51+
)
52+
FetchContent_MakeAvailable(qtmesh_onnxruntime)
53+
54+
# The archive extracts to a single top-level dir with include/ and lib/.
55+
set(QTMESH_ONNX_ROOT "${qtmesh_onnxruntime_SOURCE_DIR}")
56+
set(QTMESH_ONNX_INCLUDE_DIR "${QTMESH_ONNX_ROOT}/include")
57+
58+
# Resolve the actual shared-lib path. Prebuilt layouts vary slightly across
59+
# platforms (versioned symlinks on *nix, lib/*.dll on Windows), so glob for it
60+
# rather than hardcoding a single name.
61+
file(GLOB _ort_libs
62+
"${QTMESH_ONNX_ROOT}/lib/${_ort_libname}"
63+
"${QTMESH_ONNX_ROOT}/lib/libonnxruntime*.dylib"
64+
"${QTMESH_ONNX_ROOT}/lib/libonnxruntime.so*"
65+
"${QTMESH_ONNX_ROOT}/lib/onnxruntime.dll")
66+
if(NOT _ort_libs)
67+
message(FATAL_ERROR "ENABLE_ONNX: could not locate the ONNX Runtime shared library "
68+
"under ${QTMESH_ONNX_ROOT}/lib")
69+
endif()
70+
list(GET _ort_libs 0 QTMESH_ONNX_RUNTIME_LIB)
71+
set(QTMESH_ONNX_RUNTIME_LIB "${QTMESH_ONNX_RUNTIME_LIB}"
72+
CACHE FILEPATH "Path to the ONNX Runtime shared library to ship next to the binary" FORCE)
73+
74+
add_library(qtmesh_onnx SHARED IMPORTED GLOBAL)
75+
set_target_properties(qtmesh_onnx PROPERTIES
76+
IMPORTED_LOCATION "${QTMESH_ONNX_RUNTIME_LIB}"
77+
INTERFACE_INCLUDE_DIRECTORIES "${QTMESH_ONNX_INCLUDE_DIR}")
78+
if(WIN32)
79+
# On Windows the import library is needed for linking.
80+
file(GLOB _ort_implib "${QTMESH_ONNX_ROOT}/lib/onnxruntime.lib")
81+
if(_ort_implib)
82+
list(GET _ort_implib 0 _ort_implib0)
83+
set_target_properties(qtmesh_onnx PROPERTIES IMPORTED_IMPLIB "${_ort_implib0}")
84+
endif()
85+
endif()
86+
87+
message(STATUS "ONNX Runtime ${QTMESH_ONNX_VERSION}: ${QTMESH_ONNX_RUNTIME_LIB}")

qml/TexturePropertiesPanel.qml

Lines changed: 180 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,27 @@ GroupBox {
279279
// the texture to certain materials isn't working yet).
280280
RowLayout {
281281
Layout.fillWidth: true
282-
Item { Layout.fillWidth: true }
282+
// #404: synthesize normal/roughness/height from the current diffuse.
283+
// Only shown on an ONNX build; disabled until a real texture is set.
284+
ThemedButton {
285+
id: pbrSynthBtn
286+
text: "Generate PBR maps from diffuse"
287+
visible: MaterialEditorQML.aiPbrAvailable()
288+
enabled: MaterialEditorQML.textureName !== ""
289+
&& MaterialEditorQML.textureName !== "*Select a texture*"
290+
onClicked: {
291+
pbrStatus.text = "Synthesizing PBR maps…"
292+
MaterialEditorQML.generatePbrFromDiffuse()
293+
}
294+
}
295+
ThemedLabel {
296+
id: pbrStatus
297+
Layout.fillWidth: true
298+
visible: MaterialEditorQML.aiPbrAvailable()
299+
text: ""
300+
elide: Text.ElideRight
301+
}
302+
Item { Layout.fillWidth: true; visible: !MaterialEditorQML.aiPbrAvailable() }
283303
ThemedButton {
284304
text: "Save Texture As…"
285305
// Bind to textureName (a NOTIFY property) so the
@@ -295,6 +315,165 @@ GroupBox {
295315
MaterialEditorQML.exportCurrentTexture(dest)
296316
}
297317
}
318+
319+
Connections {
320+
target: MaterialEditorQML
321+
function onPbrSynthCompleted(result) {
322+
pbrStatus.text = result.fromCache ? "PBR maps ready (cached)."
323+
: "PBR maps generated."
324+
}
325+
function onPbrSynthError(err) { pbrStatus.text = "PBR: " + err }
326+
}
327+
}
328+
329+
// ── PBR slots (multi-slot) view ─────────────────────────────────────
330+
// Expandable section listing every texture unit of the current pass
331+
// (albedo/normal_map/roughness/…), each with its own preview + picker.
332+
// Offered only for PBR-named materials; collapsed by default so the
333+
// single-texture controls above stay the simple path.
334+
GroupBox {
335+
id: pbrSlotsGroup
336+
Layout.fillWidth: true
337+
// Visibility tracks `units` (updated by the Connections block below)
338+
// rather than the Q_INVOKABLE isPbrMaterial() — the latter has no
339+
// NOTIFY signal so a method-call binding would go stale when slots
340+
// are added/removed at runtime (e.g. after Generate PBR).
341+
property var pbrSlotNames: ["albedo", "diffuse_map", "normal_map",
342+
"roughness", "metallic", "ao", "emissive"]
343+
visible: units.some(function(u) { return pbrSlotNames.indexOf(u) >= 0 })
344+
topPadding: 22
345+
leftPadding: 6
346+
rightPadding: 6
347+
bottomPadding: 6
348+
349+
property bool expanded: false
350+
property var units: MaterialEditorQML.textureUnitList
351+
352+
title: ""
353+
background: Rectangle {
354+
color: "transparent"
355+
Rectangle {
356+
anchors.top: parent.top; anchors.left: parent.left
357+
anchors.right: parent.right; height: 1
358+
color: MaterialEditorQML.borderColor
359+
}
360+
}
361+
label: Item {
362+
width: pbrSlotsGroup.width; height: 22
363+
ThemedLabel {
364+
id: pbrSlotsHeader
365+
anchors.left: parent.left; topPadding: 4
366+
text: (pbrSlotsGroup.expanded ? "" : "")
367+
+ "PBR Texture Slots (" + pbrSlotsGroup.units.length + ")"
368+
font.bold: true
369+
}
370+
MouseArea {
371+
anchors.fill: parent
372+
cursorShape: Qt.PointingHandCursor
373+
onClicked: pbrSlotsGroup.expanded = !pbrSlotsGroup.expanded
374+
}
375+
}
376+
377+
// Refresh the slot list when bindings/selection change.
378+
Connections {
379+
target: MaterialEditorQML
380+
function onTextureUnitsChanged() {
381+
pbrSlotsGroup.units = MaterialEditorQML.textureUnitList
382+
slotRepeater.bump++
383+
}
384+
function onTextureUnitListChanged() {
385+
pbrSlotsGroup.units = MaterialEditorQML.textureUnitList
386+
slotRepeater.bump++
387+
}
388+
}
389+
390+
ColumnLayout {
391+
anchors.fill: parent
392+
spacing: 6
393+
visible: pbrSlotsGroup.expanded
394+
395+
Repeater {
396+
id: slotRepeater
397+
model: pbrSlotsGroup.units
398+
property int bump: 0 // force preview re-eval on rebind
399+
400+
RowLayout {
401+
Layout.fillWidth: true
402+
spacing: 8
403+
property int slotIndex: index
404+
405+
// Slot preview thumbnail
406+
Rectangle {
407+
width: 48; height: 48
408+
color: MaterialEditorQML.panelColor
409+
border.color: MaterialEditorQML.borderColor
410+
border.width: 1
411+
Image {
412+
anchors.fill: parent; anchors.margins: 2
413+
fillMode: Image.PreserveAspectFit
414+
cache: false
415+
source: {
416+
slotRepeater.bump; // dependency
417+
var p = MaterialEditorQML.texturePreviewPathForUnit(index)
418+
return p !== "" ? p + "?v=" + slotRepeater.bump : ""
419+
}
420+
}
421+
}
422+
423+
ColumnLayout {
424+
Layout.fillWidth: true
425+
spacing: 2
426+
ThemedLabel {
427+
text: modelData // the slot/unit name
428+
font.bold: true
429+
font.pixelSize: 11
430+
}
431+
ThemedLabel {
432+
Layout.fillWidth: true
433+
elide: Text.ElideMiddle
434+
font.pixelSize: 10
435+
color: MaterialEditorQML.disabledTextColor
436+
text: {
437+
slotRepeater.bump;
438+
var t = MaterialEditorQML.textureNameForUnit(index)
439+
return t !== "" ? t : "(empty)"
440+
}
441+
}
442+
}
443+
444+
// Pick an already-loaded texture for this slot.
445+
ThemedComboBox {
446+
Layout.preferredWidth: 150
447+
property var opts: ["— pick texture —"].concat(MaterialEditorQML.getAvailableTextures())
448+
model: opts
449+
currentIndex: 0
450+
onActivated: function(i) {
451+
if (i > 0) {
452+
MaterialEditorQML.setTextureForUnit(slotIndex, opts[i])
453+
currentIndex = 0
454+
}
455+
}
456+
}
457+
458+
// Browse for a file to bind into this slot.
459+
ThemedButton {
460+
text: "Load file…"
461+
onClicked: {
462+
var p = MaterialEditorQML.openFileDialog()
463+
if (p && p.length > 0)
464+
MaterialEditorQML.loadTextureFileForUnit(slotIndex, p)
465+
}
466+
}
467+
}
468+
}
469+
470+
ThemedLabel {
471+
visible: pbrSlotsGroup.units.length === 0
472+
text: "This material has no texture units."
473+
font.pixelSize: 10
474+
color: MaterialEditorQML.disabledTextColor
475+
}
476+
}
298477
}
299478

300479
// Texture Coordinates Group

0 commit comments

Comments
 (0)