Skip to content

Commit 5d67603

Browse files
feat(multimodule): per-module split + tw link — the killer feature over genuinely separate binaries (#140) (#203)
Item 2 of the construction sequence (owner: "2 yes"). `module Name { … }` blocks (the grammar's v1.2 form) now drive a real per-module split: `parse_modules` parses each block standalone, then re-parses importers **seeded with the producer's actual region schemas** — so cross-module accesses lower against the producer's real field offsets and instance counts, not the expected subset's packing. `tw build --split` emits one wasm per module and `tw link` runs the ADR-0007 link-graph pass over the built binaries: ``` tw build game.twasm --split → game.physics.wasm + game.ai.wasm + game.render.wasm tw link game.*.wasm → CERTIFIED ai imports physics.Entity CERTIFIED render imports physics.Entity ``` The examples/02 killer feature now runs over genuinely separate binaries instead of a merged parse (fixture: `tests/fixtures/multimodule/game.twasm`). Also lands #140's remaining deliverable — the producer-emitted callee/caller differential pair, committed under `tests/fixtures/producer_pair/` (clean caller accepted, double-caller rejected). **181 workspace tests, 0 failures.** 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent 55af7bb commit 5d67603

10 files changed

Lines changed: 519 additions & 18 deletions

File tree

crates/typed-wasm-codegen/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,5 @@ cargo test -p typed-wasm-codegen
5858

5959
- **#127** — codegen coverage across all 10 levels × all 6 examples (and the front-end → IR JSON seam).
6060
- **#130** — promote the round-trip tests into the ECHIDNA property corpus.
61+
- **Per-module split****done**: `module Name { … }` blocks parse into separate modules (`parse_modules`; importers seeded with the producer's actual schema), `tw build --split` emits one wasm per module, `tw link` certifies the graph (`tests/multimodule_split.rs`, `tests/fixtures/multimodule/game.twasm`).
6162
- **L13 positive-form / region-imports****done** (issue #140): `import region … from "…" { … }` parses into `Module::region_imports`, emits the `typedwasm.region-imports` carrier (proposal 0003 `[accepted]` / ADR-0007), and `verify_link_graph` certifies cross-module schema agreement (`tests/example02.rs`).

crates/typed-wasm-codegen/src/bin/tw.rs

Lines changed: 124 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@
44
//! `tw` — the typed-wasm build CLI (codegen v0).
55
//!
66
//! Usage:
7-
//! tw build <file.twasm> [-o <out.wasm>]
7+
//! tw build <file.twasm> [-o <out>] [--emit wasm|wat|both] [--split]
8+
//! tw link <a.wasm> <b.wasm> …
89
//!
9-
//! v0 supports only the example-01 schema; general `.twasm` front-end →
10-
//! IR lowering is tracked in ADR-0004 and issue #127.
10+
//! `--split` emits one wasm per top-level `module Name { … }` block
11+
//! (`<out>.<module>.wasm`); `link` runs the L13 cross-module link-graph
12+
//! pass (ADR-0007) over already-built modules, naming each by its file
13+
//! stem, and reports certificates / violations.
1114
1215
use std::path::Path;
1316
use std::process::ExitCode;
@@ -16,6 +19,7 @@ fn main() -> ExitCode {
1619
let args: Vec<String> = std::env::args().collect();
1720
match args.get(1).map(String::as_str) {
1821
Some("build") => build(&args[2..]),
22+
Some("link") => link(&args[2..]),
1923
Some("--version") | Some("-V") => {
2024
println!("tw {}", env!("CARGO_PKG_VERSION"));
2125
ExitCode::SUCCESS
@@ -33,7 +37,8 @@ fn main() -> ExitCode {
3337
}
3438

3539
fn usage() {
36-
eprintln!("usage: tw build <file.twasm> [-o <out>] [--emit wasm|wat|both]");
40+
eprintln!("usage: tw build <file.twasm> [-o <out>] [--emit wasm|wat|both] [--split]");
41+
eprintln!(" tw link <a.wasm> <b.wasm> ...");
3742
}
3843

3944
/// What `tw build` emits.
@@ -65,6 +70,7 @@ fn build(rest: &[String]) -> ExitCode {
6570
let mut input: Option<String> = None;
6671
let mut output: Option<String> = None;
6772
let mut emit = Emit::Wasm;
73+
let mut split = false;
6874
let mut i = 0;
6975
while i < rest.len() {
7076
let arg = rest[i].as_str();
@@ -96,6 +102,7 @@ fn build(rest: &[String]) -> ExitCode {
96102
return ExitCode::FAILURE;
97103
}
98104
},
105+
"--split" => split = true,
99106
s if !s.starts_with('-') => input = Some(s.to_string()),
100107
other => {
101108
eprintln!("tw build: unknown option '{other}'");
@@ -119,8 +126,11 @@ fn build(rest: &[String]) -> ExitCode {
119126
}
120127
};
121128

122-
// Try to parse the .twasm file using the Rust parser (issue #127).
123-
// This parser handles the paint-type schemas and example-01.
129+
if split {
130+
return build_split(&input, &src, output.as_deref(), emit);
131+
}
132+
133+
// Parse via the canonical Rust front-end (ADR-0006).
124134
let bytes = match typed_wasm_codegen::parser::parse_module(&src) {
125135
Ok(module) => {
126136
let bytes = typed_wasm_codegen::emit(&module);
@@ -171,6 +181,114 @@ fn build(rest: &[String]) -> ExitCode {
171181
ExitCode::SUCCESS
172182
}
173183

184+
/// `tw build --split`: one output per `module Name { … }` block.
185+
fn build_split(input: &str, src: &str, output: Option<&str>, emit: Emit) -> ExitCode {
186+
let modules = match typed_wasm_codegen::parser::parse_modules(src) {
187+
Ok(m) => m,
188+
Err(e) => {
189+
eprintln!("tw build: parse error in '{input}': {e}");
190+
return ExitCode::FAILURE;
191+
}
192+
};
193+
let base = output.unwrap_or(input);
194+
let stem = Path::new(base)
195+
.with_extension("")
196+
.to_string_lossy()
197+
.into_owned();
198+
let mut wrote = Vec::new();
199+
for (name, module) in &modules {
200+
let bytes = typed_wasm_codegen::emit(module);
201+
if let Err(diagnostics) = typed_wasm_codegen::self_verify(module) {
202+
for msg in diagnostics {
203+
eprintln!("tw build: self-verify warning [{name}]: {msg}");
204+
}
205+
}
206+
if emit.wants_wasm() {
207+
let path = format!("{stem}.{name}.wasm");
208+
if let Err(e) = std::fs::write(&path, &bytes) {
209+
eprintln!("tw build: cannot write '{path}': {e}");
210+
return ExitCode::FAILURE;
211+
}
212+
wrote.push(format!("{path} ({} bytes)", bytes.len()));
213+
}
214+
if emit.wants_wat() {
215+
let path = format!("{stem}.{name}.wat");
216+
let text = typed_wasm_codegen::wat(&bytes);
217+
if let Err(e) = std::fs::write(&path, text.as_bytes()) {
218+
eprintln!("tw build: cannot write '{path}': {e}");
219+
return ExitCode::FAILURE;
220+
}
221+
wrote.push(path);
222+
}
223+
}
224+
eprintln!(
225+
"tw build: split {} module(s): {}",
226+
modules.len(),
227+
wrote.join(" + ")
228+
);
229+
eprintln!("tw build: check cross-module schema agreement with `tw link <files…>`.");
230+
ExitCode::SUCCESS
231+
}
232+
233+
/// `tw link`: the L13 positive-form link-graph pass (ADR-0007) over
234+
/// built modules. Each module's wasm-level name is its file stem's last
235+
/// dot-segment (`game.physics.wasm` → `physics`).
236+
fn link(files: &[String]) -> ExitCode {
237+
if files.is_empty() {
238+
eprintln!("tw link: no modules given");
239+
usage();
240+
return ExitCode::FAILURE;
241+
}
242+
let mut named: Vec<(String, Vec<u8>)> = Vec::new();
243+
for f in files {
244+
let bytes = match std::fs::read(f) {
245+
Ok(b) => b,
246+
Err(e) => {
247+
eprintln!("tw link: cannot read '{f}': {e}");
248+
return ExitCode::FAILURE;
249+
}
250+
};
251+
let stem = Path::new(f)
252+
.with_extension("")
253+
.file_name()
254+
.map(|s| s.to_string_lossy().into_owned())
255+
.unwrap_or_else(|| f.clone());
256+
let name = stem.rsplit('.').next().unwrap_or(&stem).to_string();
257+
named.push((name, bytes));
258+
}
259+
let graph: Vec<(&str, &[u8])> = named
260+
.iter()
261+
.map(|(n, b)| (n.as_str(), b.as_slice()))
262+
.collect();
263+
match typed_wasm_verify::verify_link_graph(&graph) {
264+
Ok(report) => {
265+
for c in &report.certificates {
266+
println!(
267+
"CERTIFIED {} imports {}.{}",
268+
c.consumer, c.producer, c.region_name
269+
);
270+
}
271+
for e in &report.errors {
272+
eprintln!("VIOLATION {e}");
273+
}
274+
if report.errors.is_empty() {
275+
eprintln!(
276+
"tw link: {} module(s), {} certificate(s), no violations",
277+
named.len(),
278+
report.certificates.len()
279+
);
280+
ExitCode::SUCCESS
281+
} else {
282+
ExitCode::FAILURE
283+
}
284+
}
285+
Err(e) => {
286+
eprintln!("tw link: wasm parse error: {e}");
287+
ExitCode::FAILURE
288+
}
289+
}
290+
}
291+
174292
/// `base` with its extension replaced by `ext`
175293
/// (e.g. `with_extension("foo.twasm", "wasm")` -> `"foo.wasm"`).
176294
fn with_extension(base: &str, ext: &str) -> String {

crates/typed-wasm-codegen/src/parser.rs

Lines changed: 140 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,127 @@
2020
use crate::{Field, FieldTy, Memory, Module, PtrKind, Region, Scalar, Wty};
2121
use std::collections::HashMap;
2222

23-
/// Parse a .twasm source file into a Module IR.
23+
/// Parse a .twasm source file into a single merged Module IR.
24+
/// `module Name [isolated] { … }` blocks are flattened into one module
25+
/// (the historical single-module view the round-trip corpus uses);
26+
/// [`parse_modules`] gives the per-module split.
2427
pub fn parse_module(src: &str) -> Result<Module, String> {
25-
let parser = Parser::new(src);
26-
parser.parse_module()
28+
let fragments = slice_modules(src);
29+
if fragments.len() == 1 {
30+
return Parser::new(src).parse_module();
31+
}
32+
let merged: String = fragments
33+
.into_iter()
34+
.map(|(_, frag)| frag)
35+
.collect::<Vec<_>>()
36+
.join("\n");
37+
Parser::new(&merged).parse_module()
38+
}
39+
40+
/// Parse a multi-module `.twasm` source into one Module per top-level
41+
/// `module Name [isolated] { … }` block (declarations outside any block
42+
/// form the `"main"` module, omitted when empty). Two passes: modules
43+
/// parse independently, then any module importing a region whose
44+
/// producer is in the same file is RE-parsed with the producer's actual
45+
/// region schemas seeded in — so cross-module accesses lower against
46+
/// the producer's real layout, not the expected subset's packing.
47+
pub fn parse_modules(src: &str) -> Result<Vec<(String, Module)>, String> {
48+
let fragments = slice_modules(src);
49+
let mut parsed: Vec<(String, Module, Vec<u64>, String)> = Vec::new();
50+
for (name, frag) in fragments {
51+
let (module, instances) = Parser::new(&frag).parse_with_instances()?;
52+
if name == "main"
53+
&& module.regions.is_empty()
54+
&& module.funcs.is_empty()
55+
&& module.region_imports.is_empty()
56+
{
57+
continue; // no default-module content
58+
}
59+
parsed.push((name, module, instances, frag));
60+
}
61+
62+
// Pass 2: seed importers with in-file producers' actual schemas.
63+
let producer_snapshot: Vec<(String, Module, Vec<u64>)> = parsed
64+
.iter()
65+
.map(|(n, m, i, _)| (n.clone(), m.clone(), i.clone()))
66+
.collect();
67+
for (_, module, _, frag) in parsed.iter_mut() {
68+
if module.region_imports.is_empty() {
69+
continue;
70+
}
71+
let mut seeds: Vec<(Region, u64)> = Vec::new();
72+
for imp in &module.region_imports {
73+
if let Some((_, producer, instances)) = producer_snapshot
74+
.iter()
75+
.find(|(n, _, _)| *n == imp.producer_module)
76+
{
77+
if let Some(idx) =
78+
producer.regions.iter().position(|r| r.name == imp.region_name)
79+
{
80+
seeds.push((
81+
producer.regions[idx].clone(),
82+
instances.get(idx).copied().unwrap_or(1),
83+
));
84+
}
85+
}
86+
}
87+
if !seeds.is_empty() {
88+
let mut p = Parser::new(frag);
89+
for (region, instances) in seeds {
90+
p.region_map.insert(region.name.clone(), p.regions.len());
91+
p.regions.push(region);
92+
p.region_instances.push(instances);
93+
}
94+
*module = p.parse_module()?;
95+
}
96+
}
97+
98+
Ok(parsed.into_iter().map(|(n, m, _, _)| (n, m)).collect())
99+
}
100+
101+
/// Split source at top-level `module Name [isolated] { … }` blocks into
102+
/// `(name, fragment)` pairs; text outside any block accumulates into a
103+
/// `"main"` fragment (kept even when blank so `parse_module` can join).
104+
fn slice_modules(src: &str) -> Vec<(String, String)> {
105+
let mut out: Vec<(String, String)> = Vec::new();
106+
let mut default_frag = String::new();
107+
let mut scanner = Parser::new(src);
108+
let mut plain_start = 0usize;
109+
loop {
110+
scanner.skip_whitespace();
111+
if scanner.pos >= src.len() {
112+
break;
113+
}
114+
if scanner.peek_word("module") {
115+
let block_start = scanner.pos;
116+
scanner.pos += "module".len();
117+
let name = scanner.parse_ident();
118+
scanner.skip_whitespace();
119+
let _ = scanner.try_keyword("isolated");
120+
scanner.skip_whitespace();
121+
if !name.is_empty() && scanner.try_char('{') {
122+
default_frag.push_str(&src[plain_start..block_start]);
123+
let body_start = scanner.pos;
124+
scanner.skip_to_brace_close();
125+
// skip_to_brace_close consumed through the closing `}`.
126+
let body_end = scanner.pos.saturating_sub(1).max(body_start);
127+
out.push((name, src[body_start..body_end].to_string()));
128+
plain_start = scanner.pos;
129+
continue;
130+
}
131+
// `module X;` or malformed: leave for the plain fragment.
132+
scanner.pos = block_start + "module".len();
133+
}
134+
// Advance one declaration: consume through it so we never split
135+
// inside a nested brace (fn bodies, regions, …).
136+
scanner.skip_declaration();
137+
}
138+
default_frag.push_str(&src[plain_start..]);
139+
if out.is_empty() {
140+
return vec![("main".into(), default_frag)];
141+
}
142+
out.insert(0, ("main".into(), default_frag));
143+
out
27144
}
28145

29146
/// A simple hand-written parser for .twasm syntax.
@@ -63,7 +180,15 @@ impl<'a> Parser<'a> {
63180
}
64181
}
65182

66-
fn parse_module(mut self) -> Result<Module, String> {
183+
fn parse_module(self) -> Result<Module, String> {
184+
self.parse_with_instances().map(|(m, _)| m)
185+
}
186+
187+
/// As [`Self::parse_module`], additionally returning the per-region
188+
/// declared instance counts (needed to seed importers in
189+
/// [`parse_modules`] so their scans iterate the producer's real
190+
/// instance array).
191+
fn parse_with_instances(mut self) -> Result<(Module, Vec<u64>), String> {
67192
while self.pos < self.src.len() {
68193
self.skip_whitespace();
69194
if self.pos >= self.src.len() {
@@ -88,14 +213,17 @@ impl<'a> Parser<'a> {
88213
}
89214
}
90215

91-
Ok(Module {
92-
regions: self.regions,
93-
memory: self.memory,
94-
imports: self.imports,
95-
funcs: self.funcs,
96-
ownership: self.ownership,
97-
region_imports: self.region_imports,
98-
})
216+
Ok((
217+
Module {
218+
regions: self.regions,
219+
memory: self.memory,
220+
imports: self.imports,
221+
funcs: self.funcs,
222+
ownership: self.ownership,
223+
region_imports: self.region_imports,
224+
},
225+
self.region_instances,
226+
))
99227
}
100228

101229
fn peek_word(&mut self, word: &str) -> bool {

0 commit comments

Comments
 (0)