emit: separate the parser library from the CLI runner (Target ≠ CLI)

The portable targets baked a stdin → CST-JSON `main` into the emitted source — harness
scaffolding the gate needs to execute and diff a compiled go/rust binary, but not part
of the parser. So `emitParser(grammar, tsTarget|goTarget|rustTarget)` shipped a CLI app,
not a parser; the consumption model also diverged from jsTarget (an importable library).

Split the two:
- `emitParser(grammar, target)` now emits a parser LIBRARY only — a `parse` entry, no I/O,
  no `main`. All four targets are consistent: you import/embed and call `parse`.
- `target.emitRunner()` (new, optional on Target) emits the stdin → JSON CLI wrapper. The
  gate assembles library + runner itself: ts/rust append it (one file); Go writes it as a
  separate runner.go in the same `package main` (Go forbids a second import block after
  declarations), so the parser file keeps only its own imports.

Net: `emitParser(grammar, tsTarget)` is now directly `import { parse }` → a plain CST tree
— the "import a module, get a tree" path that was previously missing (jsTarget is a
low-level arena API; the portable TS gives a materialized CST). The CLI is the harness's
concern, where it belongs.

portable-targets 16 grammars ×3 (library + runner, compiled & run), full suite 42/42.
README updated to show library usage.

Author: johnsoncodehk

README.md

@@ -340,15 +340,18 @@ const Regex = token(seq(
 
 ### The emitted parser need not be JS — Go, Rust, native
 
-The grammar also derives a **standalone parser in another language**. [`emitParser(grammar, target)`](src/emit.ts) runs one analysis into one language-agnostic IR, and each `Target` renders it — including its own regex-free lexer (`emitParser` reuses `emitLexer(grammar, target)`), so the output has no dependency on the JS runtime and compiles offline:
+The grammar also derives a **parser library in another language**. [`emitParser(grammar, target)`](src/emit.ts) runs one analysis into one language-agnostic IR, and each `Target` renders it — including its own regex-free lexer (`emitParser` reuses `emitLexer(grammar, target)`), so the output has no dependency on the JS runtime and compiles offline. What it emits is a **library** — a `parse` entry, no I/O — that you embed and call:
 
 ```ts
-import { emitParser, goTarget, rustTarget } from './src/emit.ts';
+import { emitParser, tsTarget, goTarget, rustTarget } from './src/emit.ts';
 
-writeFileSync('parser.go', emitParser(grammar, goTarget));   // `go build`, no deps
-writeFileSync('parser.rs', emitParser(grammar, rustTarget)); // `rustc`, no crates
+writeFileSync('parser.mts', emitParser(grammar, tsTarget));  // import { parse } → a plain CST
+writeFileSync('parser.go',  emitParser(grammar, goTarget));  // a Go parser, no deps
+writeFileSync('parser.rs',  emitParser(grammar, rustTarget)); // a Rust parser, no crates
 ```
 
+The CLI shape `test/portable-targets.ts` runs (stdin → CST JSON) is a *harness* wrapper — `target.emitRunner()`, appended by the gate to make the library executable — not part of the parser.
+
 The proof is the full languages: the real [`javascript.ts`](javascript.ts) and [`typescript.ts`](typescript.ts) grammars — including the `[Await]/[Yield]` fork, left recursion, the regex/division and template state machines, arrow functions, and the TS type grammar — emit to **TypeScript, Go, and Rust**, and every CST is byte-identical to the reference interpreter. [`test/portable-targets.ts`](test/portable-targets.ts) compiles and runs all three for sixteen grammars (the two real languages plus focused fixtures) on every CI run. The Rust output reaches [oxc](https://github.com/oxc-project/oxc) throughput and the Go output beats [tsgo](https://github.com/microsoft/typescript-go) on the same corpus (an arena keeps both near zero-allocation). Byte-based Go/Rust use UTF-8 offsets — identical to the JS interpreter's for ASCII; non-ASCII offset units differ inherently.
 
 ## Adding a language

src/emit.ts

@@ -16,7 +16,11 @@ export interface Target {
   name: string;
   ext: string;                                                  // emitted file extension (no dot)
   emitLexer(grammar: CstGrammar): string | null;               // null ⇒ runtime-lexer fallback (jsTarget markup/indent grammars)
-  emitParser(grammar: CstGrammar, lexerSrc: string | null): string;   // the parser, embedding `lexerSrc`
+  emitParser(grammar: CstGrammar, lexerSrc: string | null): string;   // the parser LIBRARY (exports a `parse` entry; no I/O)
+  // A standalone CLI harness (stdin → CST JSON) APPENDED to the library to make it
+  // executable — needed to run the compiled go/rust (and ts) parsers for verification.
+  // Not part of the parser. jsTarget omits it (it is consumed by `import`, not as a CLI).
+  emitRunner?(): string;
 }
 
 export function emitLexer(grammar: CstGrammar, target: Target): string | null {

src/target-go.ts

@@ -316,16 +316,15 @@ export const goTarget: Target = {
 \treturn finish("$template", sb, t.Off)
 }
 ` : '';
-    return `// GENERATED by emit-portable.ts (goTarget) — parser for grammar "${ir.grammarName}".
+    return `// GENERATED by emit-portable.ts (goTarget) — parser LIBRARY for grammar "${ir.grammarName}".
+// Exposes parseOnce(src) + writeJSON. The CLI runner (stdin → CST JSON) is goTarget.emitRunner()
+// — a SEPARATE runner.go in this same \`package main\` (Go forbids a second import block after
+// declarations, so it can't be appended inline). Together they \`go build\` into an executable.
 package main
 
 import (
 \t"fmt"
-\t"io"
-\t"os"
-\t"strconv"
 \t"strings"
-\t"time"
 )
 
 type Tok struct {
@@ -432,6 +431,22 @@ func parseOnce(src string) int32 {
 \tnodes = nodes[:0]; kids = kids[:0]; scratch = scratch[:0]
 \treturn parse${ir.entry}()
 }
+`;
+  },
+  emitRunner(): string {
+    return `// CLI runner (harness only): stdin -> CST JSON + a self-bench mode. A SEPARATE file in the
+// same package main as the parser library; NOT part of the parser. The gate writes it as
+// runner.go so the package builds into an executable.
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
 
 func main() {
 \tdata, _ := io.ReadAll(os.Stdin)

src/target-rust.ts

@@ -415,24 +415,39 @@ fn write_json(c: &Cst, out: &mut String) {
     out.push_str(&format!("],\\"offset\\":{},\\"end\\":{}}}", c.offset, c.end));
 }
 
+// The library entry: lex + parse the whole input, returning the CST root (None on a parse
+// error / trailing input). No I/O — see emitRunner() for the stdin -> JSON wrapper.
+fn parse(src: &str) -> Option<Cst> {
+    let toks = lex(src);
+    let n = toks.len();
+    let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), src };
+    match p.parse_${ir.entry}() {
+        Some(root) if p.pos == n => Some(root),
+        _ => None,
+    }
+}
+`;
+  },
+  emitRunner(): string {
+    return `
+// CLI runner (harness only): stdin -> CST JSON + a self-bench mode. Appended to the parser
+// library by the gate (same file/crate, so it calls \`parse\`/\`write_json\` directly); NOT part
+// of the parser.
 fn main() {
+    use std::io::Read;
     let mut src = String::new();
     std::io::stdin().read_to_string(&mut src).unwrap();
     // Self-bench: a numeric arg N times the lex+parse loop and prints ms/iteration.
     if let Some(iters) = std::env::args().nth(1).and_then(|a| a.parse::<u64>().ok()) {
-        // black_box on the input + result so the optimizer can't elide the lex/parse.
-        for _ in 0..3 { let toks = lex(std::hint::black_box(&src)); let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), src: &src }; std::hint::black_box(p.parse_${ir.entry}()); }
+        for _ in 0..3 { std::hint::black_box(parse(std::hint::black_box(&src))); }
         let t = std::time::Instant::now();
-        for _ in 0..iters { let toks = lex(std::hint::black_box(&src)); let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), src: &src }; std::hint::black_box(p.parse_${ir.entry}()); }
+        for _ in 0..iters { std::hint::black_box(parse(std::hint::black_box(&src))); }
         println!("{:.4}", t.elapsed().as_secs_f64() * 1000.0 / iters as f64);
         return;
     }
-    let toks = lex(&src);
-    let n = toks.len();
-    let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), src: &src };
-    match p.parse_${ir.entry}() {
-        Some(root) if p.pos == n => { let mut out = String::new(); write_json(&root, &mut out); print!("{}", out); }
-        _ => { eprintln!("parse error (pos {}/{})", p.pos, n); std::process::exit(1); }
+    match parse(&src) {
+        Some(root) => { let mut out = String::new(); write_json(&root, &mut out); print!("{}", out); }
+        None => { eprintln!("parse error"); std::process::exit(1); }
     }
 }
 `;

src/target-ts.ts

@@ -296,8 +296,8 @@ export const tsTarget: Target = {
   return { rule: '$template', children, offset: children[0].offset, end: children[children.length - 1].end };
 }
 ` : '';
-    return `// GENERATED by emit-portable.ts (tsTarget) — parser for grammar "${ir.grammarName}".
-import { readFileSync } from 'node:fs';
+    return `// GENERATED by emit-portable.ts (tsTarget) — parser LIBRARY for grammar "${ir.grammarName}" (exports \`parse\`).
+// The CLI runner (stdin → CST JSON) is a SEPARATE piece — tsTarget.emitRunner(), appended by the harness.
 
 type Tok = { kind: string; text: string; off: number; end: number; nl: boolean };
 type Leaf = { tokenType: string; offset: number; end: number };
@@ -363,16 +363,24 @@ function altLit(opts: [string, string][], kids: Cst[]): boolean {
 
 ${matchTemplate}${ruleFns}
 
-const src = readFileSync(0, 'utf8');
-_src = src;
-toks = lex(src);
-pos = 0;
-const root = parse${ir.entry}();
-if (root === null || pos !== toks.length) {
-  process.stderr.write('parse error (pos ' + pos + '/' + toks.length + ')\\n');
-  process.exit(1);
+// The library entry: lex + parse the whole input, returning the CST root (or null on a
+// parse error / trailing input). No I/O — see emitRunner() for the stdin → JSON wrapper.
+export function parse(src: string): Cst | null {
+  _src = src;
+  toks = lex(src);
+  pos = 0;
+  const root = parse${ir.entry}();
+  return root !== null && pos === toks.length ? root : null;
 }
-process.stdout.write(JSON.stringify(root));
+`;
+  },
+  emitRunner(): string {
+    return `// CLI runner (harness only): stdin → CST JSON. Appended to the parser library by the gate;
+// NOT part of the emitted parser. The import is hoisted, so it may follow the library code.
+import { readFileSync } from 'node:fs';
+const _root = parse(readFileSync(0, 'utf8'));
+if (_root === null) { process.stderr.write('parse error\\n'); process.exit(1); }
+process.stdout.write(JSON.stringify(_root));
 `;
   },
 };

test/portable-targets.ts

@@ -242,20 +242,25 @@ for (const c of CASES) {
   mkdirSync(dir, { recursive: true });
   const runners: Array<{ label: string; run: (src: string) => Outcome }> = [];
 
+  // emitParser is the parser LIBRARY (exports a `parse` entry, no I/O); the executable CLI
+  // runner (stdin → CST JSON) is the target's emitRunner(), assembled in HERE — the harness's
+  // job, not the parser's. ts/rust append the runner to the library (one file); Go needs it as
+  // a SEPARATE runner.go in the same `package main` (its import rules forbid appending inline).
   const tsFile = `${dir}/p.ts`;
-  writeFileSync(tsFile, emitParser(grammar, tsTarget));
+  writeFileSync(tsFile, emitParser(grammar, tsTarget) + (tsTarget.emitRunner?.() ?? ''));
   runners.push({ label: 'typescript', run: (src) => runProc('node', [tsFile], src) });
 
   if (HAS_GO && !c.tsOnly) {
     const gdir = `${dir}/go`; mkdirSync(gdir, { recursive: true });
-    writeFileSync(`${gdir}/main.go`, emitParser(grammar, goTarget));
+    writeFileSync(`${gdir}/parser.go`, emitParser(grammar, goTarget));
+    writeFileSync(`${gdir}/runner.go`, goTarget.emitRunner?.() ?? '');
     writeFileSync(`${gdir}/go.mod`, 'module p\n\ngo 1.21\n');
     execFileSync('go', ['build', '-o', `${gdir}/p`, '.'], { cwd: gdir, stdio: 'pipe' });
     runners.push({ label: 'go', run: (src) => runProc(`${gdir}/p`, [], src) });
   }
   if (HAS_RUST && !c.tsOnly) {
     const rfile = `${dir}/main.rs`;
-    writeFileSync(rfile, emitParser(grammar, rustTarget));
+    writeFileSync(rfile, emitParser(grammar, rustTarget) + (rustTarget.emitRunner?.() ?? ''));
     execFileSync('rustc', ['-O', '-A', 'warnings', rfile, '-o', `${dir}/pr`], { stdio: 'pipe' });
     runners.push({ label: 'rust', run: (src) => runProc(`${dir}/pr`, [], src) });
   }