Skip to content

Commit 4625edd

Browse files
physicsrobclaude
andcommitted
Exporters return OnnxArtifact; token extra_metadata; max_layers=400, verbose=False defaults
- OnnxArtifact (export.py, next to the sidecar plumbing): frozen dataclass of paths + build scalars (kind, n_layers, per_layer_n_heads, d, d_head, cache_stride, cache_window; token kind adds d_embed/vocab_size). Built strictly AFTER export from paths/scalars — holds no graph/weights/exporter state (the streaming memory bound is a stated hard invariant). .load() and .debug_session(output_node, pos_encoding) lazy-import their targets to avoid the onnx_load/onnx_debug import cycle. - compile_to_onnx and compile_headless_to_onnx now return it (were -> None; nothing consumed the old return). - compile_to_onnx gains extra_metadata, written nested under the sidecar 'extra' key (mirrors the headless exporter; top-level keys unchanged — additive only). Surfaced via OnnxTokenModule.metadata. - compile_headless_to_onnx gains optimize=0, threaded to forward_compile (compile_to_onnx already had it). - Defaults flipped on both exporters: max_layers 200->400 (pure cap at optimize=0; no optimize>=1 caller relies on the old default — doom passes it explicitly), verbose True->False (user decision). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 119e684 commit 4625edd

1 file changed

Lines changed: 115 additions & 10 deletions

File tree

torchwright/compiler/export.py

Lines changed: 115 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
"""Compile a torchwright graph to a KV-cached ONNX model.
22
3-
Two exporters, symmetric:
3+
Two exporters, symmetric, both returning an :class:`OnnxArtifact`
4+
(paths + small build metadata; ``artifact.load()`` /
5+
``artifact.debug_session(...)``):
46
57
compile_to_onnx(output_node, pos_encoding, embedding, path, ...)
68
Token I/O: token_ids -> logits. Sidecar format
79
``torchwright.token.v1`` carries the vocab. Consumer:
8-
:mod:`torchwright.compiler.repl`.
10+
``OnnxTokenModule`` via
11+
:func:`torchwright.compiler.onnx_load.load_onnx`.
912
1013
compile_headless_to_onnx(output_node, pos_encoding, path, ...)
1114
Float I/O: inputs -> outputs. Sidecar format
1215
``torchwright.headless.v1`` carries the alphabetically-ordered
13-
input column names. Consumer:
14-
:mod:`torchwright.compiler.onnx_load`.
16+
input column names. Consumer: ``OnnxHeadlessModule`` via
17+
:func:`torchwright.compiler.onnx_load.load_onnx`.
1518
1619
Both speak the STATIC-cache prefill/decode protocol (the vanilla
1720
HF-StaticCache / vLLM pattern, chosen so ONNX Runtime can capture a CUDA
@@ -168,6 +171,63 @@ def debug_meta_path_for(onnx_path: str) -> str:
168171
return base + ".debug.json"
169172

170173

174+
@dataclass(frozen=True)
175+
class OnnxArtifact:
176+
"""Paths + small build metadata for a finished ONNX export.
177+
178+
Returned by :func:`compile_to_onnx` and :func:`compile_headless_to_onnx`
179+
so consumers stop reconstructing paths and build facts by convention
180+
(re-reading layer counts out of the ONNX file, rebuilding
181+
``d_embed``/``vocab_size`` from globals).
182+
183+
HARD INVARIANT: built strictly from paths and scalars AFTER export
184+
completes — holds no graph, no weights, no exporter state. The
185+
exporters' streaming memory bound (one dense layer's worth of weights
186+
in RAM regardless of depth) is sacred; this handle must never grow a
187+
field that would anchor the compiled model in memory.
188+
"""
189+
190+
path: str
191+
meta_path: str
192+
debug_path: Optional[str] # None when debug_sidecar=False
193+
kind: str # "token" | "headless"
194+
n_layers: int
195+
per_layer_n_heads: Tuple[int, ...] # tuple copy, not the exporter's live list
196+
d: int
197+
d_head: int
198+
cache_stride: int
199+
cache_window: Optional[int]
200+
d_embed: Optional[int] = None # token kind only
201+
vocab_size: Optional[int] = None # token kind only
202+
203+
def load(self, providers=None):
204+
"""Load the artifact via :func:`torchwright.compiler.onnx_load.load_onnx`.
205+
206+
Returns an ``OnnxTokenModule`` or ``OnnxHeadlessModule`` per
207+
``kind``.
208+
"""
209+
# Function-level import: onnx_load imports from this module at
210+
# module level, so importing it at the top would be a cycle.
211+
from torchwright.compiler.onnx_load import load_onnx
212+
213+
return load_onnx(self.path, providers=providers)
214+
215+
def debug_session(self, output_node, pos_encoding, providers=None):
216+
"""Open a :class:`torchwright.debug.onnx_debug.OnnxDebugSession`.
217+
218+
``output_node``/``pos_encoding`` must come from the same
219+
deterministic graph-construction code the export used (the
220+
session fingerprint-checks the rebuild).
221+
"""
222+
# Function-level import: onnx_debug imports from this module at
223+
# module level, so importing it at the top would be a cycle.
224+
from torchwright.debug.onnx_debug import OnnxDebugSession
225+
226+
return OnnxDebugSession(
227+
self.path, output_node, pos_encoding, providers=providers
228+
)
229+
230+
171231
def _write_debug_sidecar(
172232
onnx_path: str,
173233
*,
@@ -943,18 +1003,23 @@ def compile_headless_to_onnx(
9431003
d: int = 1024,
9441004
d_head: int = 16,
9451005
max_seq_len: int = 512,
946-
max_layers: int = 200,
947-
verbose: bool = True,
1006+
max_layers: int = 400,
1007+
verbose: bool = False,
9481008
extra_metadata: Optional[dict] = None,
9491009
d_hidden: Optional[int] = None,
9501010
trim_heads: bool = True,
1011+
optimize: int = 0,
9511012
assume_zero_init: bool = True,
9521013
cache_stride: Optional[int] = None,
9531014
cache_window: Optional[int] = None,
9541015
debug_sidecar: bool = True,
955-
) -> None:
1016+
) -> OnnxArtifact:
9561017
"""Compile a float-I/O graph to a KV-cached ONNX model.
9571018
1019+
Returns an :class:`OnnxArtifact` (paths + small build metadata;
1020+
``artifact.load()`` for the runtime, ``artifact.debug_session(...)``
1021+
for the debug surface).
1022+
9581023
Writes three files:
9591024
``<output_path>`` — the ONNX model
9601025
``<stem>.meta.json`` — ``{"format": "torchwright.headless.v1",
@@ -1042,6 +1107,7 @@ def compile_headless_to_onnx(
10421107
on_layer_compiled=on_layer_compiled,
10431108
d_hidden=d_hidden,
10441109
trim_heads=trim_heads,
1110+
optimize=optimize,
10451111
assume_zero_init=assume_zero_init,
10461112
)
10471113
t_compile = time.perf_counter() - t0
@@ -1240,6 +1306,19 @@ def add(op, ins, outs, **attrs):
12401306
print(f"Wrote {output_path} ({model_size:,} bytes)")
12411307
print(f"Wrote {meta_path}")
12421308

1309+
return OnnxArtifact(
1310+
path=output_path,
1311+
meta_path=meta_path,
1312+
debug_path=debug_meta_path_for(output_path) if debug_sidecar else None,
1313+
kind="headless",
1314+
n_layers=n_layers,
1315+
per_layer_n_heads=tuple(per_layer_n_heads),
1316+
d=d,
1317+
d_head=d_head,
1318+
cache_stride=cache_stride_resolved,
1319+
cache_window=cache_window,
1320+
)
1321+
12431322

12441323
def compile_to_onnx(
12451324
output_node: Node,
@@ -1249,18 +1328,27 @@ def compile_to_onnx(
12491328
d: int = 1024,
12501329
d_head: int = 16,
12511330
max_seq_len: int = 512,
1252-
max_layers: int = 200,
1253-
verbose: bool = True,
1331+
max_layers: int = 400,
1332+
verbose: bool = False,
12541333
trim_heads: bool = True,
12551334
optimize: int = 0,
12561335
assume_zero_init: bool = True,
12571336
d_hidden: Optional[int] = None,
1337+
extra_metadata: Optional[dict] = None,
12581338
cache_stride: Optional[int] = None,
12591339
cache_window: Optional[int] = None,
12601340
debug_sidecar: bool = True,
1261-
) -> None:
1341+
) -> OnnxArtifact:
12621342
"""Compile a token-I/O graph to a KV-cached ONNX model.
12631343
1344+
Returns an :class:`OnnxArtifact` (paths + small build metadata;
1345+
``artifact.load()`` for the runtime, ``artifact.debug_session(...)``
1346+
for the debug surface).
1347+
1348+
``extra_metadata`` is a free-form dict written under the sidecar's
1349+
``"extra"`` key (surfaced via ``OnnxTokenModule.metadata``), mirroring
1350+
the headless exporter; top-level sidecar keys are unchanged.
1351+
12641352
Writes three files:
12651353
``<output_path>`` — the ONNX model
12661354
``<stem>.meta.json`` — ``{"format": "torchwright.token.v1",
@@ -1526,6 +1614,8 @@ def add(op, ins, outs, **attrs):
15261614
# Windowed-cache protocol discriminator: bindings must be exactly
15271615
# C + n_new wide (see the module docstring).
15281616
token_meta["cache_window"] = int(cache_window)
1617+
if extra_metadata:
1618+
token_meta["extra"] = dict(extra_metadata)
15291619
meta_path = _write_meta(output_path, token_meta)
15301620

15311621
if debug_sidecar:
@@ -1561,6 +1651,21 @@ def add(op, ins, outs, **attrs):
15611651
print(f"Wrote {output_path} ({model_size:,} bytes)")
15621652
print(f"Wrote {meta_path}")
15631653

1654+
return OnnxArtifact(
1655+
path=output_path,
1656+
meta_path=meta_path,
1657+
debug_path=debug_meta_path_for(output_path) if debug_sidecar else None,
1658+
kind="token",
1659+
n_layers=n_layers,
1660+
per_layer_n_heads=tuple(per_layer_n_heads),
1661+
d=d,
1662+
d_head=d_head,
1663+
cache_stride=cache_stride_resolved,
1664+
cache_window=cache_window,
1665+
d_embed=d_embed,
1666+
vocab_size=int(vocab_size),
1667+
)
1668+
15641669

15651670
# ---------------------------------------------------------------------------
15661671
# In-process headless callable: a thin adapter over HeadlessTransformer.compute()

0 commit comments

Comments
 (0)