Two more structure bugs from the widened oracle; valid-only contract

Bug #2 - statement-position function merged with a following call: longest-
match let the expression arm win whenever a member/call tail made it LONGER
("function f(){}" + newline + "(g)()" became ONE IIFE-style expression
statement; tsc keeps a declaration + a separate statement). Fix: the ES2023
14.5 ExpressionStatement lookahead restriction - the expression arm may not
begin with function / async function. "class" is deliberately NOT guarded yet:
the class-DECLARATION arm is narrower than tsc's (extends-expression heritage,
bare ';' class elements, decorator placements), so 31 tsc-valid corpus files
still rely on the class-EXPRESSION fallback - widening the declaration arm is
the named prerequisite. The guard flips exactly 3 corpus files to reject, all
tsc-INVALID (template-typed params, "function* gen" without parens): FN=0
held. Rest parameters also gained their checker-not-parser tail
("...b = init", "...b?: T") so the declaration arm carries what the expression
fallback used to.

Bug #3 (pre-existing, all builds) - bare for-in heads: ForHead's
no-declaration arm parsed its target Expr WITHOUT the no-"in" exclusion, so
"for (key in obj)" swallowed the "in" inside an in-LED, the arm failed, and
the statement fell back to a CALL parse "for(...)" with "for" as an
identifier. Same exclude as binding initializers on the target.

The interpreter's maxPos moves to advance-based tracking (mirroring the
emitted engine's relocation) - the new reject inputs exposed that the two
engines' farthest-position semantics had silently diverged; reject messages
are engine-identical again (468 files).

ts-ast-verify gains the valid-only contract: files where tsc itself reports
parse errors are SKIPPED (error-recovery shapes are each parser's own policy,
not the grammar contract) - parserharness/parserRealSource7 fall under it; the
lowering still gained their missing shapes (NewTarget index form, old-style
type assertion). Gate corpus widened to fixSignatureCaching (2,952 nodes) +
parserindenter (3,168), both node-identical to tsc. 28/28 gates; 18,805-file
interp=emit; conformance 5383/5659 (baseline minus the 3 correct rejects).

Author: johnsoncodehk

javascript.ts

@@ -354,7 +354,7 @@ const Param = rule($ => {
     [Ident, opt('=', Expr)],
     [BindingPattern, opt('=', Expr)],
     // a rest element can never validly be a reserved word (`...while`), so guarding it is FN-safe.
-    ['...', alt([notReserved, Ident], BindingPattern)],   // rest
+    ['...', alt([notReserved, Ident], BindingPattern), opt('=', Expr)],   // rest (an initializer is a CHECKER error in tsc, not a parse error)
   );
   return [
     [opt(DecoratorExpr), body],
@@ -371,7 +371,11 @@ const ForHead = rule($ => {
       [alt('in', 'of'), Expr],
     )],
     [opt(Expr, many(',', Expr)), ...cTail],   // C-style, no declaration: `for (i=0; …; …)` / `for (;;)`
-    [Expr, alt('in', 'of'), Expr],            // for-in/of, no declaration: `for (x of xs)`
+    // for-in/of, no declaration: `for (x of xs)`. The target Expr parses in a no-`in`
+    // context (same exclude as binding initializers): the `in` belongs to the for-head,
+    // not to an in-LED inside the target — without it `for (key in obj)` swallowed the
+    // `in`, the arm failed, and the statement fell back to a CALL parse `for(...)`.
+    [exclude('in', Expr), alt('in', 'of'), Expr],
   ];
 });
 
@@ -400,7 +404,17 @@ const Stmt = rule($ => [
   ['with', '(', Expr, ')', $],
   [opt('await'), 'using', sep(Binding, ','), opt(';')],
   Decl,
-  [Expr, many(',', Expr), opt(';')],
+  // ExpressionStatement lookahead restriction (ES2023 §14.5): a statement may not
+  // begin with `function` / `async function` — those are declarations at statement
+  // level. Without this guard, longest-match lets the expression arm win whenever a
+  // call/member tail makes it LONGER (`function f(){}\n(g)()` merged into one
+  // IIFE-style expression statement; tsc keeps them separate). `{` needs no guard
+  // (the Block alternative ties in length and wins as the first-listed alternative).
+  // `class` is NOT guarded yet: the class-DECLARATION arm is narrower than tsc's
+  // (extends-expression heritage, bare `;` class elements, decorator placements), so
+  // 31 tsc-valid corpus files still rely on the class-EXPRESSION fallback — widen the
+  // declaration arm first, then guard.
+  [not(alt('function', ['async', 'function'])), Expr, many(',', Expr), opt(';')],
 ]);
 
 // ── Declarations ──

src/gen-parser.ts

@@ -760,7 +760,7 @@ export function createParser(grammar: CstGrammar) {
   function parse(source: string, entryRule?: string): CstNode {
     const tokens = tokenize(source);
     let pos = 0;
-    let maxPos = 0;   // farthest token index the parser ever attempted to read (diagnostic)
+    let maxPos = 0;   // farthest token index ever ADVANCED past (diagnostic; updated at the pos++ sites, mirroring the emitted engine so reject messages stay engine-identical)
     // Packrat memo for pratt/left-recursive rules (Expr, Type, …): cache the
     // parse result + end position by start position, so backtracking doesn't
     // re-parse the same rule at the same spot. Sound because those rules reset
@@ -777,7 +777,6 @@ export function createParser(grammar: CstGrammar) {
     let parseLimit = -1;
 
     function peek(): Token | null {
-      if (pos > maxPos) maxPos = pos;
       if (parseLimit >= 0 && pos >= parseLimit) return null;
       return tokens[pos] ?? null;
     }
@@ -794,14 +793,14 @@ export function createParser(grammar: CstGrammar) {
       if (isKeywordLiteral(value)) {
         // Keyword literal: match against Ident token with same text
         if (tok.type !== '' && tokenNames.has(tok.type) && tok.text === value) {
-          pos++;
+          if (++pos > maxPos) maxPos = pos;
           return { tokenType: '$keyword', offset: tok.offset, end: tok.offset + tok.text.length };
         }
         return null;
       }
       // Punctuation literal
       if (tok.type === '' && tok.text === value) {
-        pos++;
+        if (++pos > maxPos) maxPos = pos;
         return { tokenType: '$punct', offset: tok.offset, end: tok.offset + tok.text.length };
       }
       // Split multi-`>` tokens: `>>`, `>>>`, `>>=`, `>>>=` can yield a single `>`
@@ -812,7 +811,7 @@ export function createParser(grammar: CstGrammar) {
           { type: '', text: rest, offset: tok.offset + 1, k: 0, t: 0, newlineBefore: false, commentBefore: false, multilineFlowBefore: false },
         );
         memo.clear();   // splice shifts later token indices → memo entries are stale
-        pos++;
+        if (++pos > maxPos) maxPos = pos;
         return { tokenType: '$punct', offset: tok.offset, end: tok.offset + 1 };
       }
       return null;
@@ -823,7 +822,7 @@ export function createParser(grammar: CstGrammar) {
       const tok = peek();
       if (!tok) return null;
       if (tok.type === name) {
-        pos++;
+        if (++pos > maxPos) maxPos = pos;
         return { tokenType: name, offset: tok.offset, end: tok.offset + tok.text.length };
       }
       return null;
@@ -842,12 +841,12 @@ export function createParser(grammar: CstGrammar) {
       const tok = peek();
       if (!tok) return null;
       if (tok.type === templateTokenName) {
-        pos++;
+        if (++pos > maxPos) maxPos = pos;
         return { tokenType: templateTokenName, offset: tok.offset, end: tok.offset + tok.text.length };
       }
       if (tok.type === '$templateHead') {
         const children: CstChild[] = [];
-        pos++;
+        if (++pos > maxPos) maxPos = pos;
         children.push({ tokenType: '$templateHead', offset: tok.offset, end: tok.offset + tok.text.length });
         const interpRule = currentPrattContext ?? findExprRule();
         while (true) {
@@ -856,12 +855,12 @@ export function createParser(grammar: CstGrammar) {
           const next = peek();
           if (!next) break;
           if (next.type === '$templateMiddle') {
-            pos++;
+            if (++pos > maxPos) maxPos = pos;
             children.push({ tokenType: '$templateMiddle', offset: next.offset, end: next.offset + next.text.length });
             continue;
           }
           if (next.type === '$templateTail') {
-            pos++;
+            if (++pos > maxPos) maxPos = pos;
             children.push({ tokenType: '$templateTail', offset: next.offset, end: next.offset + next.text.length });
             break;
           }
@@ -1041,7 +1040,7 @@ export function createParser(grammar: CstGrammar) {
             const key = tok.text;
             const info = prefixOps.get(key);
             if (info) {
-              pos++;
+              if (++pos > maxPos) maxPos = pos;
               const opLeaf: CstLeaf = { tokenType: '$operator', offset: tok.offset, end: tok.offset + tok.text.length };
               const rhs = parsePratt(rule, info.rbp);
               if (rhs && pos > bestNudPos) {
@@ -1136,7 +1135,7 @@ export function createParser(grammar: CstGrammar) {
         if (info && info.lbp > minBp) {
           if (info.position === 'postfix') {
             if (!tailClosed) {                                   // can't postfix an update expr (`a++ --`)
-              pos++;
+              if (++pos > maxPos) maxPos = pos;
               const opLeaf: CstLeaf = { tokenType: '$operator', offset: tok.offset, end: tok.offset + tok.text.length };
               lhs = { rule: rule.name, children: [lhs, opLeaf], offset: lhs.offset, end: opLeaf.end };
               tailClosed = true;
@@ -1157,7 +1156,7 @@ export function createParser(grammar: CstGrammar) {
                 return null;
               }
             }
-            pos++;
+            if (++pos > maxPos) maxPos = pos;
             const opLeaf: CstLeaf = { tokenType: '$operator', offset: tok.offset, end: tok.offset + tok.text.length };
             const rhs = parsePratt(rule, info.rbp);
             if (rhs) {

test/ts-ast-lowering.ts

@@ -164,6 +164,10 @@ function lowerExpr(n: CstNode): Ast {
     if (t0 === '...' && c.length === 2 && ruleIs(c[1], 'Expr')) {
       return ast('SpreadElement', n.offset, n.end, [lowerExpr(c[1])]);
     }
+    // Old-style type assertion: ['<', Type, '>', Expr]
+    if (t0 === '<' && c.length === 4 && ruleIs(c[1], 'Type') && ruleIs(c[3], 'Expr')) {
+      return ast('TypeAssertionExpression', n.offset, n.end, [lowerType(c[1]), lowerExpr(c[3])]);
+    }
     // Parenthesized vs arrow: both arms start '('.
     // PAIN(5): two different grammar ALTERNATIVES with the same first token reach the
     // consumer as child-shape puzzles — '(' Expr ')' vs '(' Param* ')' '=>' …; the
@@ -287,6 +291,10 @@ function lowerNewTarget(n: CstNode): Ast {
   if (ruleIs(c[0], 'NewTarget') && c.length === 3 && isLeaf(c[2])) {
     return ast('PropertyAccessExpression', n.offset, n.end, [lowerNewTarget(c[0]), ast('Identifier', c[2].offset, c[2].end)]);
   }
+  // index form: NewTarget '[' Expr ']'  (`new benchmarks[i]()`)
+  if (ruleIs(c[0], 'NewTarget') && c.length === 4 && leafIs(c[1], '[') && ruleIs(c[2], 'Expr')) {
+    return ast('ElementAccessExpression', n.offset, n.end, [lowerNewTarget(c[0]), lowerExpr(c[2])]);
+  }
   if (c.length === 1 && ruleIs(c[0], 'Expr')) return lowerExpr(c[0]);
   throw new Unlowered('NewTarget shape', n.offset);
 }

test/ts-ast-verify.ts

@@ -54,7 +54,16 @@ function compare(mine: Ast, theirs: ts.Node, sf: ts.SourceFile, st: Stats): void
   for (let i = 0; i < tc.length; i++) compare(mine.children[i], tc[i], sf, st);
 }
 
-function run(name: string, code: string): { ok: boolean; line: string; samples: string[] } {
+function run(name: string, code: string): { ok: boolean; skipped?: boolean; line: string; samples: string[] } {
+  // Structural contract: VALID files only. When tsc itself reports parse errors, its
+  // tree is an error-RECOVERY shape (each parser's recovery strategy is its own) —
+  // comparing structures there compares recovery policies, not the grammar.
+  {
+    const probe = ts.createSourceFile('f.ts', code, ts.ScriptTarget.Latest, false, ts.ScriptKind.TS) as ts.SourceFile & { parseDiagnostics: unknown[] };
+    if (probe.parseDiagnostics.length > 0) {
+      return { ok: true, skipped: true, line: `${name}: SKIPPED (tsc reports ${probe.parseDiagnostics.length} parse error(s) — recovery shapes are out of contract)`, samples: [] };
+    }
+  }
   let cst;
   try { cst = parser.parse(code); }
   catch (e) { return { ok: false, line: `${name}: MONOGRAM REJECT ${(e as Error).message.slice(0, 60)}`, samples: [] }; }
@@ -113,7 +122,10 @@ const SNIPPETS: [string, string][] = [
   ['comma-empty', ';;'],
 ];
 
-const CORPUS_FILE = '/tmp/ts-repo/tests/cases/conformance/parser/ecmascript5/RealWorld/parserindenter.ts';
+const CORPUS_FILES = [
+  '/tmp/ts-repo/tests/cases/conformance/parser/ecmascript5/RealWorld/parserindenter.ts',
+  '/tmp/ts-repo/tests/cases/conformance/fixSignatureCaching.ts',
+];
 const files = process.argv.slice(2);
 let pass = 0, fail = 0;
 if (files.length > 0) {
@@ -131,8 +143,9 @@ if (files.length > 0) {
     for (const s of r.samples) console.log('    ', s);
     r.ok ? pass++ : fail++;
   }
-  if (existsSync(CORPUS_FILE)) {
-    const r = run('parserindenter.ts', readFileSync(CORPUS_FILE, 'utf-8'));
+  for (const cf of CORPUS_FILES) {
+    if (!existsSync(cf)) continue;
+    const r = run(cf.split('/').pop()!, readFileSync(cf, 'utf-8'));
     console.log((r.ok ? '✓ ' : '✗ ') + r.line);
     for (const s of r.samples) console.log('    ', s);
     r.ok ? pass++ : fail++;

typescript.ts

@@ -270,7 +270,7 @@ const Param = rule($ => {
     [BindingPattern, ...tail],
     // a rest element, by contrast, can never validly be a reserved word (`...while`),
     // and `...this` is invalid too, so guarding the rest name is FN-safe.
-    ['...', alt([notReserved, Ident], BindingPattern), opt('?'), opt(':', Type)],   // rest
+    ['...', alt([notReserved, Ident], BindingPattern), opt('?'), opt(':', Type), opt('=', Expr)],   // rest (`?`/initializer are CHECKER errors in tsc, not parse errors)
   );
   return [
     ['this', ':', Type],
@@ -292,7 +292,11 @@ const ForHead = rule($ => {
       [alt('in', 'of'), Expr],
     )],
     [opt(Expr, many(',', Expr)), ...cTail],   // C-style, no declaration: `for (i=0; …; …)` / `for (;;)`
-    [Expr, alt('in', 'of'), Expr],            // for-in/of, no declaration: `for (x of xs)`
+    // for-in/of, no declaration: `for (x of xs)`. The target Expr parses in a no-`in`
+    // context (same exclude as binding initializers): the `in` belongs to the for-head,
+    // not to an in-LED inside the target — without it `for (key in obj)` swallowed the
+    // `in`, the arm failed, and the statement fell back to a CALL parse `for(...)`.
+    [exclude('in', Expr), alt('in', 'of'), Expr],
   ];
 });
 
@@ -321,7 +325,17 @@ const Stmt = rule($ => [
   ['with', '(', Expr, ')', $],
   [opt('await'), 'using', sep(Binding, ','), opt(';')],
   Decl,
-  [Expr, many(',', Expr), opt(';')],
+  // ExpressionStatement lookahead restriction (ES2023 §14.5): a statement may not
+  // begin with `function` / `async function` — those are declarations at statement
+  // level. Without this guard, longest-match lets the expression arm win whenever a
+  // call/member tail makes it LONGER (`function f(){}\n(g)()` merged into one
+  // IIFE-style expression statement; tsc keeps them separate). `{` needs no guard
+  // (the Block alternative ties in length and wins as the first-listed alternative).
+  // `class` is NOT guarded yet: the class-DECLARATION arm is narrower than tsc's
+  // (extends-expression heritage, bare `;` class elements, decorator placements), so
+  // 31 tsc-valid corpus files still rely on the class-EXPRESSION fallback — widen the
+  // declaration arm first, then guard.
+  [not(alt('function', ['async', 'function'])), Expr, many(',', Expr), opt(';')],
 ]);
 
 // ── Type Parameters ──