emit-portable: no-in suppress, +-quantifier, sep/bracket fixes — javascript.ts now EMITS

A run of constructs that together take the real javascript.ts grammar through the
whole portable emitter end-to-end:

- no-`in` (suppress) context: a `for (binding in iterable)` head parses its binding
  with the `in` led disabled (examples/noinjs.ts, 9/9+4/4 ×3). Threads a
  suppressed-connector set consumed per led loop.
- one-or-more `+` quantifier (`x+` = `x x*`) — the last buildIR throw; with it,
  javascript.ts EMITS in all three targets.
- Two latent `sep` bugs, both exposed only by the real grammar (earlier grammars
  wrapped sep in opt or never tested the shapes — the aggregate passed for the wrong
  reason): gen-parser's sep is `(element (delim element)*)?`, i.e. the WHOLE list is
  optional (empty `f()` valid) AND a trailing delimiter is allowed. sepBy now matches.
- A NUD bracket that fails now FALLS THROUGH to the next same-first-token alternative
  instead of returning null — javascript has four `new`-led NUDs.

Result: javascript.ts emits, compiles and runs in ts/go/rust, and is byte-identical to
createParser on basic JS (var/function/arrow/ternary/member-call/for-in/while/if/class/
new/template/regex/instanceof/try/switch) — 23/24 in TS, the one miss a `new a.b()`
NewTarget member-constructor CST shape. The await/yield fork (async/await) and that
new-expression edge remain. Full suite 42/42; existing gate unaffected by the shared
sep/bracket fixes.

Author: johnsoncodehk

examples/noinjs.ts

@@ -0,0 +1,35 @@
+// Exercises the no-`in` (suppress) context. In a `for (binding in iterable)` head, the
+// binding is parsed with the `in` LED DISABLED — `exclude('in', Expr)` — so the `in` belongs
+// to the for-head, not to a relational expression inside the binding. Outside a for-head, `in`
+// binds normally. The portable parser threads a suppressed-connector set into the led loop.
+import {
+  token, rule, defineGrammar, left, op, exclude,
+  seq, oneOf, range, star, many,
+} from '../src/api.ts';
+
+const idStart = oneOf(range('a', 'z'), range('A', 'Z'), '_', '$');
+const idCont = oneOf(range('a', 'z'), range('A', 'Z'), range('0', '9'), '_', '$');
+const Ident = token(seq(idStart, star(idCont)), { identifier: true, scope: 'variable' });
+const Number_ = token(seq(range('0', '9'), star(range('0', '9'))), { scope: 'constant.numeric' });
+
+const jsPrec = [left('||'), left('<', '>'), left('+', '-')];
+
+const Expr = rule(($) => [
+  Number_, Ident,
+  ['(', $, ')'],
+  [$, op, $],
+  [$, 'in', $],
+  [$, '.', Ident],
+]);
+const ForHead = rule(($) => [['for', '(', exclude('in', Expr), 'in', Expr, ')', Stmt]]);
+const Stmt = rule(($) => [ForHead, [Expr, ';']]);
+const Program = rule(($) => [many(Stmt)]);
+
+export default defineGrammar({
+  name: 'noinjs',
+  scopeName: 'source.noinjs',
+  tokens: { Ident, Number: Number_ },
+  prec: jsPrec,
+  ledPrec: [{ connector: 'in', sameAs: '<', chainRhs: true }],
+  rules: { Expr, ForHead, Stmt, Program },
+});

src/emit-portable.ts

@@ -54,7 +54,8 @@ export type Step =
   | { t: 'alt'; branches: Step[][] }                            // inline alternation of sub-sequences (backtracking)
   | { t: 'not'; steps: Step[] }                                 // zero-width negative lookahead (consumes nothing)
   | { t: 'seq'; steps: Step[] }                                 // a grouped sub-sequence (e.g. a star body `(',' Expr)`)
-  | { t: 'sameLine' };                                          // zero-width: the next token is on the same line (no preceding newline)
+  | { t: 'sameLine' }                                           // zero-width: the next token is on the same line (no preceding newline)
+  | { t: 'suppress'; connectors: string[]; steps: Step[] };     // parse the body with these LED connectors disabled (no-`in` context)
 export type Alt = Step[];
 
 export type RdRule = { kind: 'rd'; name: string; alts: Alt[] };
@@ -144,9 +145,9 @@ function buildIR(grammar: CstGrammar): ParserIR {
     switch (e.type) {
       case 'literal': return { t: 'lit', value: e.value, ttype: litTtype(e.value) };
       case 'ref': return tokenNames.has(e.name) ? { t: 'tok', name: e.name } : { t: 'rule', name: e.name };
-      case 'group': {   // transparent (ctxMode is invisible to the portable parser); only no-in `suppress` is deferred
-        if (e.suppress) throw new Error('portable: group with suppress (no-in context) not yet in scope');
+      case 'group': {   // transparent (ctxMode is invisible to the portable parser)
         const ss = altSteps(e.body);
+        if (e.suppress && e.suppress.length) return { t: 'suppress', connectors: e.suppress, steps: ss };   // no-`in` context
         return ss.length === 1 ? ss[0] : { t: 'seq', steps: ss };
       }
       case 'not': return { t: 'not', steps: altSteps(e.body) };   // zero-width negative lookahead
@@ -156,7 +157,7 @@ function buildIR(grammar: CstGrammar): ParserIR {
       case 'quantifier':
         if (e.kind === '*') return { t: 'star', step: stepOf(e.body) };
         if (e.kind === '?') return { t: 'opt', steps: altSteps(e.body) };
-        if (e.kind === '+') throw new Error("portable: '+' not yet modeled (use '*')");
+        if (e.kind === '+') return { t: 'seq', steps: [stepOf(e.body), { t: 'star', step: stepOf(e.body) }] };   // x+ = x x*
         break;
       case 'alt': {
         if (e.items.every((it) => it.type === 'literal')) {   // fast path: all-literal alternation
@@ -289,15 +290,11 @@ function buildPratt(
         continue;
       }
       if (items[0].type === 'literal') { nudBrackets.push({ first: items[0].value, steps: items.map((it) => stepOfPratt(it)) }); continue; }
-      // A single transparent group unwraps to its body (an explicit grouping of the NUD sequence).
+      // A single transparent (non-suppress) group unwraps to its body (an explicit grouping).
       let nudItems = items;
       if (items.length === 1 && items[0].type === 'group' && !items[0].suppress) {
         nudItems = items[0].body.type === 'seq' ? items[0].body.items : [items[0].body];
       }
-      // A no-`in`/suppress group is a deeper construct — defer.
-      if (nudItems.some((it) => it.type === 'group' && it.suppress)) {
-        throw new Error(`portable: Pratt NUD with suppress (no-in context) not yet in scope (rule ${name})`);
-      }
       nudSeqs.push(nudItems.map((it) => stepOfPratt(it)));   // general NUD sequence (guarded ident, class expr)
       continue;
     }
@@ -328,13 +325,15 @@ function buildPratt(
     if (e.type === 'seq') return { t: 'seq', steps: e.items.map(stepOfPratt) };
     if (e.type === 'sameLine') return { t: 'sameLine' };
     if (e.type === 'not') return { t: 'not', steps: (e.body.type === 'seq' ? e.body.items : [e.body]).map(stepOfPratt) };
+    if (e.type === 'group' && e.suppress && e.suppress.length) return { t: 'suppress', connectors: e.suppress, steps: (e.body.type === 'seq' ? e.body.items : [e.body]).map(stepOfPratt) };
     // ctxMode (await/yield) is transparent to the portable parser (no fork); unwrap the group.
-    if (e.type === 'group' && !e.capBelow && !e.suppress) {
+    if (e.type === 'group' && !e.capBelow) {
       return e.body.type === 'seq' ? { t: 'seq', steps: e.body.items.map(stepOfPratt) } : stepOfPratt(e.body);
     }
     if (e.type === 'sep') return { t: 'sep', elem: stepOfPratt(e.element), delim: e.delimiter };
     if (e.type === 'quantifier' && e.kind === '?') return { t: 'opt', steps: (e.body.type === 'seq' ? e.body.items : [e.body]).map(stepOfPratt) };
     if (e.type === 'quantifier' && e.kind === '*') return { t: 'star', step: stepOfPratt(e.body) };
+    if (e.type === 'quantifier' && e.kind === '+') return { t: 'seq', steps: [stepOfPratt(e.body), { t: 'star', step: stepOfPratt(e.body) }] };
     if (e.type === 'literal') return { t: 'lit', value: e.value, ttype: litTtype(e.value) };
     return stepOf(e);
   }

src/target-go.ts

@@ -178,6 +178,7 @@ function stepCond(s: Step): string {
     case 'not': return `func() bool { save := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); m := ${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'}; pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]; return !m }()`;
     case 'seq': return `(${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'})`;
     case 'sameLine': return `func() bool { t := peek(); return t != nil && !t.Nl }()`;
+    case 'suppress': return `func() bool { _suppressNext = map[string]bool{${s.connectors.map((c) => `${J(c)}: true`).join(', ')}}; _r := (${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'}); _suppressNext = nil; return _r }()`;
   }
 }
 
@@ -207,9 +208,9 @@ function prattRule(r: PrattRule, tpl: TplCfg | null): string {
   const bracketNud = (b: Bracket) => `\tif t.Text == ${J(b.first)} {
 \t\tsave := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
 \t\tif ${b.steps.map(stepCond).join(' && ')} { return finish(${J(r.name)}, sb, t.Off) }
-\t\tpos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]; return -1
+\t\tpos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]
 \t}`;
-  const ledArm = (b: Bracket, accessTail: boolean, lbp: number | null) => `\t\tif ${accessTail ? '!tailClosed && ' : ''}${lbp !== null ? `${lbp} > minBp && ` : ''}t.Text == ${J(b.first)} {
+  const ledArm = (b: Bracket, accessTail: boolean, lbp: number | null) => `\t\tif ${accessTail ? '!tailClosed && ' : ''}${lbp !== null ? `${lbp} > minBp && ` : ''}!_mySup[${J(b.first)}] && t.Text == ${J(b.first)} {
 \t\t\tledSave := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
 \t\t\tscratch = append(scratch, left)
 \t\t\tif ${b.steps.map(stepCond).join(' && ')} { left = finish(${J(r.name)}, sb, nodes[left].Offset); continue }
@@ -233,6 +234,7 @@ var ${r.name}POST = map[string]int{${post}}
 var ${r.name}ATOM = map[string]bool{${atoms}}
 func parse${r.name}() int32 { return ${r.name}bp(0) }
 func ${r.name}bp(minBp int) int32 {
+\t_mySup := _suppressNext; _suppressNext = nil; _ = _mySup
 \tleft := ${r.name}nud(minBp)
 \tif left < 0 { return -1 }
 \tif _capped { return left }
@@ -332,6 +334,7 @@ type bp struct{ lbp, rbp int }
 var toks []Tok
 var pos int
 var _capped bool
+var _suppressNext map[string]bool
 var nodes []Node
 var kids []int32
 var scratch []int32
@@ -379,7 +382,7 @@ func opt(body func() bool) bool {
 \tsp := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); if !body() { pos = sp; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb] }; return true
 }
 func sepBy(elem func() bool, delim string) bool {
-\tif !elem() { return false }
+\tif !elem() { return true }   // the whole separated list is optional — zero elements is valid
 \tfor {
 \t\tsp := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
 \t\tif !matchLit(delim, "$punct") { pos = sp; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]; break }

src/target-rust.ts

@@ -186,6 +186,7 @@ function stepCond(s: Step): string {
     case 'not': return `(|p: &mut Parser<'a>, k: &mut Vec<Cst>| -> bool { ${notBody(s.steps)} })(self, &mut kids)`;
     case 'seq': return `(${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'})`;
     case 'sameLine': return `matches!(self.peek(), Some(t) if !t.nl)`;
+    case 'suppress': return `{ self.suppress_next = vec![${s.connectors.map(J).join(', ')}]; let _r = (${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'}); self.suppress_next = Vec::new(); _r }`;
   }
 }
 // A backtracking inline alternation rendered as an immediately-applied closure over (p, k),
@@ -212,6 +213,7 @@ function stepCondP(s: Step): string {
     case 'not': return `(|p: &mut Parser<'a>, k: &mut Vec<Cst>| -> bool { ${notBody(s.steps)} })(p, k)`;
     case 'seq': return `(${s.steps.length ? s.steps.map(stepCondP).join(' && ') : 'true'})`;
     case 'sameLine': return `matches!(p.peek(), Some(t) if !t.nl)`;
+    case 'suppress': return `{ p.suppress_next = vec![${s.connectors.map(J).join(', ')}]; let _r = (${s.steps.length ? s.steps.map(stepCondP).join(' && ') : 'true'}); p.suppress_next = Vec::new(); _r }`;
   }
 }
 
@@ -237,9 +239,9 @@ function prattRule(r: PrattRule, tpl: TplCfg | null): string {
   const bracketNud = (b: Bracket) => `        if t.text == ${J(b.first)} {
             let save = self.pos; let mut kids: Vec<Cst> = Vec::new();
             if ${b.steps.map(stepCond).join(' && ')} { return Some(node(${J(r.name)}, kids)); }
-            self.pos = save; return None;
+            self.pos = save;   // fall through to the next NUD alternative
         }`;
-  const ledArm = (b: Bracket, accessTail: boolean, lbp: number | null) => `            if ${accessTail ? '!tail_closed && ' : ''}${lbp !== null ? `${lbp} > min_bp && ` : ''}t.text == ${J(b.first)} {
+  const ledArm = (b: Bracket, accessTail: boolean, lbp: number | null) => `            if ${accessTail ? '!tail_closed && ' : ''}${lbp !== null ? `${lbp} > min_bp && ` : ''}!my_sup.iter().any(|c| *c == ${J(b.first)}) && t.text == ${J(b.first)} {
                 let led_save = self.pos; let mut kids: Vec<Cst> = Vec::new();
                 if ${b.steps.map(stepCond).join(' && ')} {
                     let mut full = vec![left]; full.append(&mut kids);
@@ -259,6 +261,8 @@ function prattRule(r: PrattRule, tpl: TplCfg | null): string {
     fn ${r.name}_post(op: &str) -> Option<i64> { match op { ${postArms}${postArms ? ', ' : ''}_ => None } }
     fn ${r.name}_atom(kind: &str) -> bool { matches!(kind, ${atomArm || '""'}) }
     fn ${r.name}_bp(&mut self, min_bp: i64) -> Option<Cst> {
+        let my_sup = std::mem::take(&mut self.suppress_next);
+        let _ = &my_sup;
         let mut left = self.${r.name}_nud(min_bp)?;
         if self.capped { return Some(left); }
         let mut tail_closed = false;
@@ -342,7 +346,7 @@ fn node(rule: &'static str, kids: Vec<Cst>) -> Cst { let o = kids[0].offset; let
 
 ${lexer(ir)}
 
-struct Parser<'a> { toks: Vec<Tok<'a>>, pos: usize, capped: bool }
+struct Parser<'a> { toks: Vec<Tok<'a>>, pos: usize, capped: bool, suppress_next: Vec<&'static str> }
 impl<'a> Parser<'a> {
     fn peek(&self) -> Option<Tok<'a>> { if self.pos < self.toks.len() { Some(self.toks[self.pos]) } else { None } }
     fn branch(&self, rule: &'static str, kids: Vec<Cst>, save: usize) -> Cst {
@@ -367,7 +371,7 @@ impl<'a> Parser<'a> {
         let sp = self.pos; let before = kids.len(); if !body(self, kids) { self.pos = sp; kids.truncate(before); } true
     }
     fn sep_by(&mut self, elem: fn(&mut Parser<'a>, &mut Vec<Cst>) -> bool, delim: &str, kids: &mut Vec<Cst>) -> bool {
-        if !elem(self, kids) { return false; }
+        if !elem(self, kids) { return true; }   // the whole separated list is optional — zero elements is valid
         loop {
             let sp = self.pos; let before = kids.len();
             if !self.match_lit(delim, "$punct", kids) { self.pos = sp; kids.truncate(before); break; }
@@ -399,15 +403,15 @@ fn main() {
     // Self-bench: a numeric arg N times the lex+parse loop and prints ms/iteration.
     if let Some(iters) = std::env::args().nth(1).and_then(|a| a.parse::<u64>().ok()) {
         // black_box on the input + result so the optimizer can't elide the lex/parse.
-        for _ in 0..3 { let toks = lex(std::hint::black_box(&src)); let mut p = Parser { toks, pos: 0, capped: false }; std::hint::black_box(p.parse_${ir.entry}()); }
+        for _ in 0..3 { let toks = lex(std::hint::black_box(&src)); let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new() }; std::hint::black_box(p.parse_${ir.entry}()); }
         let t = std::time::Instant::now();
-        for _ in 0..iters { let toks = lex(std::hint::black_box(&src)); let mut p = Parser { toks, pos: 0, capped: false }; std::hint::black_box(p.parse_${ir.entry}()); }
+        for _ in 0..iters { let toks = lex(std::hint::black_box(&src)); let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new() }; std::hint::black_box(p.parse_${ir.entry}()); }
         println!("{:.4}", t.elapsed().as_secs_f64() * 1000.0 / iters as f64);
         return;
     }
     let toks = lex(&src);
     let n = toks.len();
-    let mut p = Parser { toks, pos: 0, capped: false };
+    let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new() };
     match p.parse_${ir.entry}() {
         Some(root) if p.pos == n => { let mut out = String::new(); write_json(&root, &mut out); print!("{}", out); }
         _ => { eprintln!("parse error (pos {}/{})", p.pos, n); std::process::exit(1); }

src/target-ts.ts

@@ -169,6 +169,7 @@ function stepCond(s: Step): string {
     case 'not': return `(() => { const sp = pos; const bk = kids.length; const m = ${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'}; pos = sp; kids.length = bk; return !m; })()`;
     case 'seq': return `(${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'})`;
     case 'sameLine': return `(() => { const t = peek(); return t !== null && !t.nl; })()`;
+    case 'suppress': return `(() => { _suppressNext = new Set([${s.connectors.map(J).join(', ')}]); const _r = (${s.steps.length ? s.steps.map(stepCond).join(' && ') : 'true'}); _suppressNext = null; return _r; })()`;
   }
 }
 
@@ -192,11 +193,11 @@ function prattRule(r: PrattRule, tpl: TplCfg | null): string {
   const bracketNud = (b: Bracket) => `    if (t.text === ${J(b.first)}) {
       const save = pos; const kids: Cst[] = [];
       if (${b.steps.map(stepCond).join(' && ')}) return node(${J(r.name)}, kids);
-      pos = save; return null;
+      pos = save;   // fall through to the next NUD alternative (e.g. another '${b.first}'-led form)
     }`;
   // Access-tail leds (member/call/index) are disabled once a postfix has closed the operand;
   // a precedence-gated led (ternary/in/instanceof) binds only when its lbp > minBp.
-  const ledArm = (b: Bracket, accessTail: boolean, lbp: number | null) => `    if (${accessTail ? '!tailClosed && ' : ''}${lbp !== null ? `${lbp} > minBp && ` : ''}t.text === ${J(b.first)}) {
+  const ledArm = (b: Bracket, accessTail: boolean, lbp: number | null) => `    if (${accessTail ? '!tailClosed && ' : ''}${lbp !== null ? `${lbp} > minBp && ` : ''}(_mySup === null || !_mySup.has(${J(b.first)})) && t.text === ${J(b.first)}) {
       const ledSave = pos; const kids: Cst[] = [left];
       if (${b.steps.map(stepCond).join(' && ')}) { left = node(${J(r.name)}, kids); continue; }
       pos = ledSave; break;
@@ -214,6 +215,7 @@ const ${r.name}_POST: Record<string, number> = ${POST};
 const ${r.name}_ATOM = ${atom};
 function parse${r.name}(): Node | null { return ${r.name}_bp(0); }
 function ${r.name}_bp(minBp: number): Node | null {
+  const _mySup = _suppressNext; _suppressNext = null;   // no-in: consume the suppressed-connector set for this led loop
   let left = ${r.name}_nud(minBp);
   if (left === null) return null;
   if (_capped) return left;   // an assignment-level arrow admits no led
@@ -293,6 +295,7 @@ ${lexer(ir)}
 let toks: Tok[] = [];
 let pos = 0;
 let _capped = false;
+let _suppressNext: Set<string> | null = null;
 function peek(): Tok | null { return pos < toks.length ? toks[pos] : null; }
 function branch(rule: string, kids: Cst[], save: number): Node {
   const offset = kids.length > 0 ? kids[0].offset : (save < toks.length ? toks[save].off : 0);
@@ -325,7 +328,7 @@ function opt(body: () => boolean, kids: Cst[]): boolean {
   const sp = pos; const before = kids.length; if (!body()) { pos = sp; kids.length = before; } return true;
 }
 function sepBy(elem: () => boolean, delim: string, kids: Cst[]): boolean {
-  if (!elem()) return false;
+  if (!elem()) return true;   // the whole separated list is optional — zero elements is valid
   for (;;) {
     const sp = pos; const before = kids.length;
     if (!matchLit(delim, '$punct', kids)) { pos = sp; kids.length = before; break; }

test/portable-targets.ts

@@ -165,6 +165,17 @@ const CASES: Case[] = [
     ],
     reject: ['a ? b;', 'a ? : c;', 'in b;', 'a instanceof;'],
   },
+  {
+    // The no-`in` (suppress) context: a `for (binding in iterable)` head parses its binding
+    // with the `in` led disabled, so `in` belongs to the for-head, not the binding.
+    grammar: 'noinjs', path: '../examples/noinjs.ts',
+    accept: [
+      'for (x in y) z;', 'x in y;', 'for (a.b in c) d;', 'a in b in c;',
+      'for ((x) in y) z;', 'for (x in y) a in b;', 'for (x in a in b) z;',
+      '(a in b);', 'for (a in b) for (c in d) e;',
+    ],
+    reject: ['for (x y) z;', 'for x in y;', 'for (in y) z;', 'for (x in) z;'],
+  },
 ];
 
 const sortKeys = (o: unknown): unknown =>