1- From 75c7df636aace01bc21fa72d40dc541174671f76 Mon Sep 17 00:00:00 2001
1+ From 508f482ef9d2ffdc64263eb690ed828f1769e62c Mon Sep 17 00:00:00 2001
22From: =?UTF-8?q?=C3=98yvind=20Harboe?= <oyvind@ascenium.com>
3- Date: Sun, 17 May 2026 15:02:44 +0200
4- Subject: [PATCH] render_gds: phase logging + dynamic drop-to-GLB-budget
5- heuristic
6- MIME-Version: 1.0
7- Content-Type: text/plain; charset=UTF-8
8- Content-Transfer-Encoding: 8bit
3+ Date: Sun, 17 May 2026 22:14:16 +0200
4+ Subject: [PATCH] render_gds: trim layerstack before Blender import for
5+ fast-turnaround viewer
96
10- Headless run on sky130hd/microwatt's 562 MB merged GDS surfaces three
11- distinct issues that the previous gltf-export path turned into an opaque
12- "appears to choke" failure:
13-
14- 1. No phase visibility.
15-
16- The action log carried no breadcrumbs about which step OOMed or
17- returned CANCELLED inside the long bpy.ops sequence (addon-register,
18- gdsii-import, gltf-addon-enable, gltf-export, html-write).
19-
20- Add a tiny `_log_phase()` helper that scrapes VmRSS/VmPeak from
21- /proc/self/status and prints one tagged line per phase boundary,
22- plus the imported mesh polygon/vertex totals and the .glb / .html
23- byte sizes. Wraps the bpy.ops calls in try/except so a Python
24- exception inside the addon prints a full traceback before
25- sys.exit, instead of being swallowed into a bare `{'CANCELLED'}`.
26-
27- 2. Int32 overflow in foreach_get for very large per-corner attributes.
28-
29- With the phase logging in place, iter 1 surfaced:
30-
31- Error: Array length mismatch (expected -464636871, got 609104952)
32- RuntimeError: internal error setting the array
33- File ".../io_scene_gltf2/blender/exp/gltf2_blender_gather_primitives_extract.py",
34- line 1405, in __get_normals
35- self.blender_mesh.corner_normals.foreach_get('vector', self.normals)
36-
37- Blender 4.2's RNA bridge stores the foreach_get element count in a
38- signed int32; microwatt's mcon layer alone has ~9.3M extruded
39- polygons × 6 faces × 4 corners ≈ 223M corner-normals = 669M float32
40- values, which overflows.
41-
42- The NORMAL attribute is gated by `self.use_normals` (`gltf_normals`
43- export setting); when False the failing path is never entered.
44- Pass `export_normals=False` plus belt-and-suspenders for the other
45- foreach_get-backed per-corner attributes (tangents, vertex colors,
46- custom attributes). model-viewer falls back to face-derived flat
47- normals, which is the right look for a chip-layout scene anyway --
48- the extruded layers are already block-like and don't benefit from
49- smooth shading.
50-
51- 3. GLB 4 GB single-file ceiling + browser per-tab memory cap.
52-
53- With normals disabled, iter 2 tripped:
54-
55- File ".../io_scene_gltf2/io/exp/gltf2_io_export.py", line 105,
56- in save_gltf
57- file.write(struct.pack("I", length))
58- struct.error: 'I' format requires 0 <= number <= 4294967295
59-
60- i.e. GLB's binary container stores total length as uint32; microwatt's
61- raw vertex+index payload is ~10 GB (mcon alone ~2.9 GB raw, licon
62- ~2.6 GB). Even if the GLB write succeeded, the base64-embedded data
63- URL in the HTML would exceed the ~2 GB per-tab memory cap mainstream
64- browsers enforce.
65-
66- iter 3 used a hardcoded sky130-specific drop set (licon, li1, mcon,
67- met1, all vias) gated by `total_polys > 30_000_000`. That worked
68- end-to-end (build succeeded; GLB 2.17 GB, HTML 2.89 GB) but is
69- PDK-specific and overshot the browser cap.
70-
71- Replace with a dynamic heuristic: rank meshes by polygon-count ×
72- bytes-per-poly, drop biggest first until estimated GLB ≤
73- _MAX_GLB_BYTES (150 MB, ≈10% of the 2 GB browser-tab cap with
74- base64-inflation margin). Bytes-per-poly is calibrated from iter 3
75- (39.7M Blender polygons -> 2.17 GB GLB -> ~55 b/poly; rounded up to
76- 60 for pessimism). Layer-name agnostic, so the same path serves
77- non-sky130 PDKs; gcd-class designs (<1M polygons) already fit under
78- the budget so the loop is a no-op for them.
79-
80- Goal here is "bling and fun" loadability in a browser, not visual
81- fidelity -- the trade-off is acceptable for an investigation-record
82- viewer. Phase log line `budget-trimmed` records the drop list and
83- estimated post-trim GLB so the next iteration can recalibrate
84- _GLB_BYTES_PER_POLY if the empirical ratio drifts.
85-
86- Signed-off-by: Øyvind Harboe <oyvind@ascenium.com>
87- Signed-off-by: Øyvind Harboe <oyvind.harboe@zylin.com>
7+ Recipe for a 'bling and fun' chip-layout viewer that finishes in
8+ seconds, fits in any browser, and still strongly resembles the
9+ real design. Replaces the after-import bpy.data.objects drop with
10+ a YAML-level layerstack trim so Blender's addon never extrudes the
11+ doomed layers in the first place.
8812---
89- tools/blendergds/render_gds.py | 187 ++++++++++++++++++++++++++++++ ---
90- 1 file changed, 171 insertions(+), 16 deletions(-)
13+ tools/blendergds/render_gds.py | 202 +++++++++++++++++++++++++++++- ---
14+ 1 file changed, 182 insertions(+), 20 deletions(-)
9115
9216diff --git a/tools/blendergds/render_gds.py b/tools/blendergds/render_gds.py
93- index 6cf4d75..d8bbbe1 100644
17+ index 6cf4d75..cbe116c 100644
9418--- a/tools/blendergds/render_gds.py
9519+++ b/tools/blendergds/render_gds.py
96- @@ -33,10 +33,82 @@ import importlib.util
20+ @@ -33,10 +33,97 @@ import importlib.util
9721 import shutil
9822 import sys
9923 import tempfile
@@ -102,51 +26,66 @@ index 6cf4d75..d8bbbe1 100644
10226 from pathlib import Path
10327
10428
105- + # Target GLB binary size for the html-export path: ~10% of the ~2 GB
106- + # per-tab memory cap mainstream browsers enforce on 64-bit Linux. After
107- + # 4/3 base64 inflation in the data: URL, this puts the resulting HTML
108- + # at ~200 MB -- "bling and fun" loadable, with plenty of headroom for
109- + # the browser to actually render the scene. Empirical, not exact.
110- + _MAX_GLB_BYTES = 150 * 1024 * 1024
29+ + # Per-PDK "bling and fun" layer preset. Each entry is the subset of
30+ + # the addon's bundled layerstack YAML we keep when trimming for a
31+ + # browser-loadable viewer:
32+ + #
33+ + # * One substrate layer (paints the chip's transistor regions as a
34+ + # coloured background, so the viewer doesn't look like a bare
35+ + # metal stack floating in space).
36+ + # * The topmost 2-3 metal layers + the highest via stack between
37+ + # them (macros and top-level routing -- the visually distinctive
38+ + # bits that say "this is microwatt", "this is gcd", etc.).
39+ + #
40+ + # The dense interconnect / contact / lower-metal layers (mcon, licon,
41+ + # li1, met1, met2, via, via2 on sky130) are dropped wholesale. They
42+ + # carry most of the polygon count but read as featureless noise at
43+ + # any zoom level that fits a 10mm² chip on a laptop screen.
44+ + #
45+ + # Effect on microwatt sky130hd: drops ~26M input polygons down to
46+ + # ~200K; the Blender per-layer extrusion that previously cost ~5-7 min
47+ + # + 16 GB RSS becomes seconds and tens of MB. The resulting GLB lands
48+ + # under ~80 MB, the base64-embedded HTML under ~110 MB -- well inside
49+ + # any browser tab's working budget.
50+ + #
51+ + # PDKs not in the table fall through to the unfiltered addon default
52+ + # stack (gcd-class designs already fit; bigger designs in other PDKs
53+ + # will need their own preset added here).
54+ + _LAYER_PRESETS = {
55+ + "SKY130": ["nwell", "met3", "via4", "met4", "met5"],
56+ + }
11157+
112- + # Bytes-per-polygon estimate, calibrated from a prior iteration on
113- + # sky130hd/microwatt that produced 39_669_907 Blender polygons in a
114- + # 2_169_758_252-byte GLB (export_normals=False, post-extrusion,
115- + # triangulated by the gltf2 exporter). ~55 bytes/poly empirically;
116- + # round up to 60 so the heuristic is slightly pessimistic.
117- + _GLB_BYTES_PER_POLY = 60
11858+
59+ + def _trim_layerstack(addon_module, addon_root, pdk_selection, tmpdir):
60+ + """Build a trimmed layerstack YAML containing only the entries listed
61+ + in _LAYER_PRESETS[pdk_selection], in the original YAML's order.
11962+
120- + def _drop_to_glb_budget(max_glb_bytes):
121- + """Greedily drop the biggest mesh objects until the estimated GLB
122- + binary size fits the budget.
63+ + Returns the path to the trimmed YAML to feed back into the addon via
64+ + `gdsii_use_custom_config=True` + `gdsii_config_path=<this>`, or None
65+ + if the PDK has no preset (fall through to the addon's default stack).
66+ + """
67+ + preset = _LAYER_PRESETS.get(pdk_selection)
68+ + if not preset:
69+ + return None
70+ + pdk_info = getattr(addon_module, "PDK_CONFIGS", {}).get(pdk_selection, {})
71+ + rel = pdk_info.get("config_path")
72+ + if not rel:
73+ + return None
74+ + yamlfile = Path(addon_root) / rel
75+ + if not yamlfile.is_file():
76+ + return None
12377+
124- + The heuristic is intentionally cheap (no trial exports): rank meshes
125- + by polygon-count × bytes-per-poly, evict largest-first. Always keeps
126- + at least one mesh -- a degenerate scene with a single mesh larger
127- + than the budget will go through to gltf-export and surface whatever
128- + failure mode that triggers, instead of producing an empty scene.
78+ + import yaml # staged via the addon's wheels
12979+
130- + Returns (list-of-(name, est_bytes, polys) tuples for dropped meshes,
131- + estimated total GLB bytes after dropping).
132- + """
133- + sized = sorted(
134- + (
135- + (len(o.data.polygons) * _GLB_BYTES_PER_POLY, o)
136- + for o in bpy.data.objects
137- + if o.type == "MESH"
138- + ),
139- + key=lambda x: x[0],
140- + reverse=True,
141- + )
142- + total = sum(sz for sz, _ in sized)
143- + dropped = []
144- + while total > max_glb_bytes and len(sized) > 1:
145- + sz, obj = sized.pop(0)
146- + dropped.append((obj.name, sz, len(obj.data.polygons)))
147- + bpy.data.objects.remove(obj, do_unlink=True)
148- + total -= sz
149- + return dropped, total
80+ + full = yaml.safe_load(yamlfile.read_text(encoding="utf-8"))
81+ + keep = set(preset)
82+ + # Preserve YAML insertion order so z-stack semantics in the addon
83+ + # (bottom-up extrusion + layered rendering) match the bundled file.
84+ + trimmed = {name: data for name, data in full.items() if name in keep}
85+ + missing = keep - trimmed.keys()
86+ + out = Path(tmpdir) / "trimmed_layerstack.yaml"
87+ + out.write_text(yaml.safe_dump(trimmed, default_flow_style=False, sort_keys=False))
88+ + return out, list(trimmed.keys()), missing
15089+
15190+
15291+ def _log_phase(phase: str, extra: str = "") -> None:
@@ -176,7 +115,7 @@ index 6cf4d75..d8bbbe1 100644
176115 def _split_argv():
177116 """Return Blender's pass-through args (everything after `--`)."""
178117 if "--" not in sys.argv:
179- @@ -166,6 +238 ,8 @@ def main():
118+ @@ -166,6 +253 ,8 @@ def main():
180119 if not wheels_dir.is_dir():
181120 sys.exit(f"render_gds.py: wheels/ dir not found under {addon_root}")
182121
@@ -185,24 +124,29 @@ index 6cf4d75..d8bbbe1 100644
185124 # Stage wheels into a temp site-packages and put it on sys.path BEFORE
186125 # importing the addon (which imports numpy/gdstk/klayout/yaml at module
187126 # scope).
188- @@ -175,6 +249 ,7 @@ def main():
127+ @@ -175,6 +264 ,7 @@ def main():
189128 try:
190129 _stage_wheels(wheels_dir, site)
191130 sys.path.insert(0, str(site))
192131+ _log_phase("wheels-staged")
193132
194133 # bpy is provided by Blender; importing it before this point would
195134 # leak partial init in --background mode.
196- @@ -182,6 +257,7 @@ def main():
135+ @@ -182,48 +272,116 @@ def main():
197136
198137 addon = _load_addon_module(addon_root)
199138 addon.register()
200139+ _log_phase("addon-registered")
201140
202141 # The addon's register_properties() declares four scene props via
203142 # the legacy `bpy.types.Scene.X = StringProperty(...)` form. In
204- @@ -192,38 +268,113 @@ def main():
205- # which is what we want anyway.
143+ # Blender 4.2 that registration is unreliable for empty-default
144+ - # string props, so we only force the one that actually deviates
145+ - # from the addon's default (the PDK key). The other three keep
146+ - # their addon defaults — use_custom_config=False, custom_*=""—
147+ - # which is what we want anyway.
148+ + # string props, so we only force the ones we actually need to
149+ + # deviate from the addon's defaults.
206150 bpy.context.scene.gdsii_pdk_selection = args.pdk
207151
208152- result = bpy.ops.import_scene.gdsii(
@@ -212,6 +156,28 @@ index 6cf4d75..d8bbbe1 100644
212156- merge_layers=True,
213157- color_scheme="realistic",
214158- )
159+ + # Pre-trim the PDK layerstack to a "bling and fun" subset. This
160+ + # MUST happen before bpy.ops.import_scene.gdsii because the
161+ + # addon's per-layer extrusion loop reads the layerstack YAML
162+ + # and pays full RAM/time cost for every entry; dropping entries
163+ + # at the YAML level is what lets a 562 MB GDS (microwatt) finish
164+ + # in seconds rather than 5-7 minutes + 16 GB RSS. No-op for
165+ + # PDKs without a preset and for designs whose full stack is
166+ + # already small enough.
167+ + trimmed = _trim_layerstack(addon, addon_root, args.pdk, tmp)
168+ + if trimmed is not None:
169+ + yaml_path, kept_layers, missing = trimmed
170+ + bpy.context.scene.gdsii_use_custom_config = True
171+ + bpy.context.scene.gdsii_config_path = str(yaml_path)
172+ + _log_phase(
173+ + "layerstack-trimmed",
174+ + extra=(
175+ + f"kept={kept_layers} "
176+ + + (f"missing={sorted(missing)} " if missing else "")
177+ + + f"yaml={yaml_path}"
178+ + ),
179+ + )
180+ +
215181+ try:
216182+ result = bpy.ops.import_scene.gdsii(
217183+ filepath=str(gds_path),
@@ -243,34 +209,6 @@ index 6cf4d75..d8bbbe1 100644
243209+ _log_phase("blend-saved", extra=f"{out_path.stat().st_size} bytes")
244210
245211 if args.out_html:
246- + # The GLB binary container stores its total length as a uint32
247- + # (struct.pack("I", length)), and the base64-embedded data URL
248- + # in the HTML must also stay under a browser tab's per-tab
249- + # memory cap. Full-stack microwatt is ~10 GB of raw vertex+
250- + # index data -- 65x over the GLB ceiling, and unreachable to
251- + # a browser even if it fit. Trim the scene to a "bling and
252- + # fun" working budget _GLB_BYTES_PER_POLY * polys ≤
253- + # _MAX_GLB_BYTES, dropping biggest meshes first. The
254- + # heuristic is layer-name agnostic so non-sky130 PDKs work
255- + # the same way; the trade-off is that on a too-large scene
256- + # the layers with the highest polygon counts (typically the
257- + # densest interconnect/contact layers like mcon, licon, li1
258- + # on sky130hd) get evicted while the visually distinctive
259- + # ones (substrate, transistor regions, upper metals with
260- + # their macros) tend to stay. gcd-class designs already fit
261- + # under the budget, so the drop loop is a no-op for them.
262- + dropped, est_glb = _drop_to_glb_budget(_MAX_GLB_BYTES)
263- + kept_objs = [o for o in bpy.data.objects if o.type == "MESH"]
264- + kept_polys = sum(len(o.data.polygons) for o in kept_objs)
265- + _log_phase(
266- + "budget-trimmed",
267- + extra=(
268- + f"max_glb={_MAX_GLB_BYTES} estimated_glb={est_glb} "
269- + f"dropped={dropped} kept_meshes={len(kept_objs)} "
270- + f"kept_polys={kept_polys}"
271- + ),
272- + )
273- +
274212 # io_scene_gltf2 ships with Blender as a core addon but is not
275213 # active under --factory-startup. Enable it before exporting.
276214 bpy.ops.preferences.addon_enable(module="io_scene_gltf2")
@@ -331,7 +269,7 @@ index 6cf4d75..d8bbbe1 100644
331269
332270 _write_html(
333271 template_path=Path(args.html_template).resolve(),
334- @@ -232,6 +383 ,10 @@ def main():
272+ @@ -232,6 +390 ,10 @@ def main():
335273 title=args.title,
336274 out_path=Path(args.out_html).resolve(),
337275 )
0 commit comments