Merge branch 'worktree-agent-ab3f7be54d1f95382'

# Conflicts:
#	examples/typescript.ts
#	src/api.ts
#	src/gen-lexer.ts
#	src/types.ts

Author: johnsoncodehk

examples/typescript.ts

@@ -1,8 +1,8 @@
 import {
   token, rule, defineGrammar,
   left, right, none,
-  op, prefix, postfix,
-  sep, opt, many, many1, alt, exclude,
+  op, prefix, postfix, sameLine,
+  sep, opt, many, many1, alt, exclude, not,
 } from '../src/api.ts';
 
 // ── Tokens ──
@@ -76,6 +76,7 @@ const TypeMember = rule($ => {
         [':', Type, ']', opt(':', Type)],                                             // index:  k: T
       )],
       [Expr, ']', opt('?'), propOrMethod],                                            // computed: expr
+      [']', opt(':', Type)],                                                          // empty index sig: []  /  []: T
     )],
     // readonly property (the readonly index signature is the bracketed branch above)
     ['readonly', Ident, opt('?'), ':', Type],
@@ -89,7 +90,7 @@ const Type = rule($ => {
   return [
     [Ident, opt('is', $)],   // T  |  type predicate `x is T`
     [$, '<', sep($, ','), '>'],
-    [$, '[', ']'],
+    [$, sameLine, '[', ']'],   // array type T[] — `[` must be on the same line (no ASI)
     [$, '|', $],
     [$, '&', $],
     ['|', $],   // leading pipe: type T = | A | B
@@ -117,7 +118,7 @@ const Type = rule($ => {
     ['unique', 'symbol'],
     ['import', '(', $, ')'],
     Template,
-    [$, '[', $, ']'],
+    [$, sameLine, '[', $, ']'],   // indexed access T[K] — `[` must be on the same line (no ASI)
     [$, '.', Ident],
   ];
 }, { type: true });
@@ -168,7 +169,10 @@ const Expr = rule($ => [
   [$, postfix],
   ['...', $],
   // instantiation / typed call / tagged template: f<T> | f<T>(…) | f<T>`…`
-  [$, '<', sep(Type, ','), '>', opt(alt(['(', sep($, ','), ')'], Template))],
+  // A bare instantiation `f<T>` (no call/tag) only when the next token can't
+  // start an expression — otherwise `<`/`>` were comparisons (`f < a, b > 7`):
+  // the same disambiguation TS makes via canFollowTypeArgumentsInExpression.
+  [$, '<', sep(Type, ','), '>', alt(['(', sep($, ','), ')'], Template, not(Expr))],
   [$, '(', sep($, ','), ')'],
   [$, '.', alt(Ident, PrivateField)],
   // optional chaining: ?.x | ?.#x | ?.(args) | ?.[i] | ?.`…`

src/api.ts

@@ -63,6 +63,7 @@ export function rule(def: RuleBody, opts?: RuleOptions): RuleRef {
 // ── Special slots for rules ──
 
 interface OpMarker { readonly __kind: 'op' }
+interface SameLineMarker { readonly __kind: 'sameLine' }
 interface PrefixSlot {
   readonly __kind: 'prefix';
   (...ops: string[]): PrefixOps;
@@ -74,10 +75,13 @@ interface PostfixSlot {
 interface PrefixOps { readonly __kind: 'prefix-ops'; ops: string[] }
 interface PostfixOps { readonly __kind: 'postfix-ops'; ops: string[] }
 
-type Marker = OpMarker | PrefixSlot | PostfixSlot;
+type Marker = OpMarker | PrefixSlot | PostfixSlot | SameLineMarker;
 
 export const op: OpMarker = { __kind: 'op' };
 
+// Zero-width "no LineTerminator here" assertion (see RuleExpr 'sameLine').
+export const sameLine: SameLineMarker = { __kind: 'sameLine' };
+
 export const prefix: PrefixSlot = Object.assign(
   (...ops: string[]): PrefixOps => ({ __kind: 'prefix-ops' as const, ops }),
   { __kind: 'prefix' as const },
@@ -132,8 +136,15 @@ class ExcludeNode {
   readonly items: Element[];
   constructor(connectors: string[], items: Element[]) { this.connectors = connectors; this.items = items; }
 }
+class NotNode {
+  readonly __kind = 'not' as const;
+  // Zero-width negative lookahead over a single element (wrap a sequence in a
+  // group/alt if needed). Matches nothing; succeeds only when `item` can't match.
+  readonly item: Element;
+  constructor(item: Element) { this.item = item; }
+}
 
-type Combinator = SepNode | OptNode | ManyNode | Many1Node | AltNode | ExcludeNode;
+type Combinator = SepNode | OptNode | ManyNode | Many1Node | AltNode | ExcludeNode | NotNode;
 
 export function sep(item: Element, delimiter: string): SepNode {
   return new SepNode(item, delimiter);
@@ -162,6 +173,12 @@ export function exclude(connectors: string | string[], ...items: Element[]): Exc
   return new ExcludeNode(typeof connectors === 'string' ? [connectors] : connectors, items);
 }
 
+// Zero-width negative lookahead: `not(x)` matches nothing and succeeds only when
+// `x` would NOT match here.
+export function not(item: Element): NotNode {
+  return new NotNode(item);
+}
+
 // ── Precedence ──
 
 interface PrecLevelDef {
@@ -255,10 +272,14 @@ function toRuleExpr(el: Element, names: Map<object, string>): RuleExpr {
       }),
     };
   }
+  if (el instanceof NotNode) {
+    return { type: 'not', body: toRuleExpr(el.item, names) };
+  }
   const marker = el as Marker;
   if (marker.__kind === 'op') return { type: 'op' };
   if (marker.__kind === 'prefix') return { type: 'prefix' };
   if (marker.__kind === 'postfix') return { type: 'postfix' };
+  if (marker.__kind === 'sameLine') return { type: 'sameLine' };
   throw new Error(`Unknown element: ${JSON.stringify(el)}`);
 }
 

src/cli.ts

@@ -94,6 +94,8 @@ function formatExpr(expr: RuleExpr): string {
     case 'alt': return expr.items.map(formatExpr).join(' | ');
     case 'quantifier': return `${formatExpr(expr.body)}${expr.kind}`;
     case 'group': return `(${formatExpr(expr.body)})`;
+    case 'not': return `not(${formatExpr(expr.body)})`;
+    case 'sameLine': return 'sameLine';
     case 'sep': return `sep(${formatExpr(expr.element)}, '${expr.delimiter}')`;
   }
 }

src/gen-lexer.ts

@@ -8,6 +8,7 @@ export interface Token {
   type: string;   // token decl name (e.g. 'Ident'), or '' for punctuation literals
   text: string;
   offset: number;
+  newlineBefore?: boolean;   // a line terminator preceded this token (drives ASI / "no LineTerminator here" rules)
 }
 
 // Build a standalone lexer from the grammar's token definitions + lexer hints.
@@ -84,11 +85,20 @@ export function createLexer(grammar: CstGrammar) {
     // that to the regex-vs-division check (consulted only when prev is `)`).
     const parenHeadStack: boolean[] = [];
     let lastCloseWasParenHead = false;
+    // A line terminator was seen since the last emitted token (skipped whitespace
+    // or comments). Stamped onto the next real token as `newlineBefore`, then
+    // cleared — so the parser can honor "no LineTerminator here" restrictions
+    // (e.g. an array/indexed-access type's `[` must be on the same line).
+    let pendingNl = false;
+    function push(t: Token): void {
+      if (pendingNl) { t.newlineBefore = true; pendingNl = false; }
+      tokens.push(t);
+    }
 
     while (pos < source.length) {
       // Skip whitespace
       const wsMatch = source.slice(pos).match(/^\s+/);
-      if (wsMatch) { pos += wsMatch[0].length; continue; }
+      if (wsMatch) { if (wsMatch[0].includes('\n')) pendingNl = true; pos += wsMatch[0].length; continue; }
 
       // Close an interpolation hole (interpClose at baseline depth) → resume the template span.
       if (templateStack.length > 0 && source.startsWith(tplInterpClose, pos)) {
@@ -99,10 +109,10 @@ export function createLexer(grammar: CstGrammar) {
           pos += tplInterpClose.length;
           const { endsWithInterp, end } = scanTemplateSpan(source, pos);
           if (endsWithInterp) {
-            tokens.push({ type: '$templateMiddle', text: source.slice(startPos, end), offset: startPos });
+            push({ type: '$templateMiddle', text: source.slice(startPos, end), offset: startPos });
             templateStack.push(0);
           } else {
-            tokens.push({ type: '$templateTail', text: source.slice(startPos, end), offset: startPos });
+            push({ type: '$templateTail', text: source.slice(startPos, end), offset: startPos });
           }
           pos = end;
           continue;
@@ -122,10 +132,10 @@ export function createLexer(grammar: CstGrammar) {
         pos += tplOpen.length;
         const { endsWithInterp, end } = scanTemplateSpan(source, pos);
         if (endsWithInterp) {
-          tokens.push({ type: '$templateHead', text: source.slice(startPos, end), offset: startPos });
+          push({ type: '$templateHead', text: source.slice(startPos, end), offset: startPos });
           templateStack.push(0);
         } else {
-          tokens.push({ type: templateTokenName!, text: source.slice(startPos, end), offset: startPos });
+          push({ type: templateTokenName!, text: source.slice(startPos, end), offset: startPos });
         }
         pos = end;
         continue;
@@ -152,7 +162,9 @@ export function createLexer(grammar: CstGrammar) {
         const m = remaining.match(tm.regex);
         if (m) {
           if (!tm.skip) {
-            tokens.push({ type: tm.name, text: m[0], offset: pos });
+            push({ type: tm.name, text: m[0], offset: pos });
+          } else if (m[0].includes('\n')) {
+            pendingNl = true;   // a skipped comment spanning a newline still terminates the previous line
           }
           pos += m[0].length;
           matched = true;
@@ -178,7 +190,7 @@ export function createLexer(grammar: CstGrammar) {
             } else if (lit === ')') {
               lastCloseWasParenHead = parenHeadStack.pop() ?? false;
             }
-            tokens.push({ type: '', text: lit, offset: pos });
+            push({ type: '', text: lit, offset: pos });
             pos += lit.length;
             matched = true;
             break;
@@ -191,7 +203,7 @@ export function createLexer(grammar: CstGrammar) {
         // missed (e.g. accented or non-Latin names). Tagged with that token's name.
         const identMatch = remaining.match(/^[\p{L}\p{Nl}_$][\p{L}\p{Nl}\p{Nd}\p{Mn}\p{Mc}\p{Pc}_$]*/u);
         if (identMatch) {
-          tokens.push({ type: identTokenName, text: identMatch[0], offset: pos });
+          push({ type: identTokenName, text: identMatch[0], offset: pos });
           pos += identMatch[0].length;
           matched = true;
         }

src/gen-parser.ts

@@ -228,6 +228,7 @@ export function createParser(grammar: CstGrammar) {
       case 'alt': return e.items.some(exprNullable);
       case 'quantifier': return e.kind === '+' ? exprNullable(e.body) : true;
       case 'group': return exprNullable(e.body);
+      case 'not': return true;                                   // zero-width assertion: consumes nothing
       case 'sep': return true;                                   // sep matches zero elements
       default: return true;                                      // op/prefix/postfix markers don't consume
     }
@@ -250,7 +251,7 @@ export function createParser(grammar: CstGrammar) {
         const acc = new Set<string>();
         for (const item of e.items) {
           if (item.type === 'prefix') return null;               // prefix op → any operator token: give up
-          if (item.type === 'op' || item.type === 'postfix') continue;  // non-consuming here
+          if (item.type === 'op' || item.type === 'postfix' || item.type === 'not' || item.type === 'sameLine') continue;  // non-consuming here
           const f = exprFirst(item);
           if (f === null) return null;
           for (const k of f) acc.add(k);
@@ -268,6 +269,7 @@ export function createParser(grammar: CstGrammar) {
         return acc;
       }
       case 'quantifier': case 'group': return exprFirst(e.body);
+      case 'not': case 'sameLine': return new Set();             // zero-width: contributes no FIRST tokens
       case 'sep': return exprFirst(e.element);
       default: return null;
     }
@@ -697,6 +699,21 @@ export function createParser(grammar: CstGrammar) {
           // rule it wraps: stage them for the next parseRule to pick up.
           if (expr.suppress && expr.suppress.length) suppressNext = new Set(expr.suppress);
           return matchExpr(expr.body);
+        case 'not': {
+          // Zero-width negative lookahead: succeed (no children) iff the body
+          // does NOT match here; never consume input either way.
+          const saved = pos;
+          const m = matchExpr(expr.body);
+          pos = saved;
+          return m === null ? [] : null;
+        }
+        case 'sameLine': {
+          // Zero-width "no LineTerminator here": succeed (no children) iff the
+          // next token isn't preceded by a newline. At EOF there's no token to
+          // continue onto, so the assertion fails (nothing same-line follows).
+          const tok = peek();
+          return tok && !tok.newlineBefore ? [] : null;
+        }
         case 'sep':
           return matchSep(expr.element, expr.delimiter);
         default:

src/gen-treesitter.ts

@@ -139,6 +139,16 @@ function renderExpr(expr: RuleExpr, ctx: GrammarJsContext): string {
     }
     case 'group':
       return renderExpr(expr.body, ctx);
+    case 'not':
+      // Zero-width negative lookahead: not expressible in a tree-sitter CFG, and
+      // it consumes nothing, so it drops to a no-op (the surrounding choice keeps
+      // the bare form, as the grammar did before the assertion was added).
+      return 'blank()';
+    case 'sameLine':
+      // Zero-width "no LineTerminator here" assertion — tree-sitter handles this
+      // class of restriction with an external scanner, not the CFG; as a CFG node
+      // it consumes nothing, so render a no-op.
+      return 'blank()';
     case 'sep': {
       // sep(elem, ',') = optional(seq(elem, repeat(seq(',', elem)), optional(',')))
       // Trailing delimiter is allowed (matches the parser's matchSep behavior).

src/types.ts

@@ -52,6 +52,19 @@ export type RuleExpr =
   | { type: 'ref'; name: string }
   | { type: 'quantifier'; body: RuleExpr; kind: '*' | '+' | '?' }
   | { type: 'group'; body: RuleExpr; suppress?: string[] }   // suppress: LED connectors disabled while parsing body (e.g. no-`in`)
+  // Zero-width negative lookahead: matches (consuming nothing) iff `body` does
+  // NOT match at the current position. Used to express disambiguations the
+  // longest-match parser can't reach by structure alone (e.g. a `<…>` type-arg
+  // list in expression position is only a bare instantiation when it isn't
+  // followed by something that starts an expression). Non-consuming → invisible
+  // to highlighting / AST shape / other generators.
+  | { type: 'not'; body: RuleExpr }
+  // Zero-width "no LineTerminator here" assertion: matches (consuming nothing)
+  // iff the NEXT token is on the same line (no preceding newline). Encodes
+  // ECMAScript/TS restricted productions like an array/indexed-access type's `[`,
+  // which must not follow a line terminator. Non-consuming → invisible to other
+  // generators (they treat it as a no-op marker).
+  | { type: 'sameLine' }
   | { type: 'sep'; element: RuleExpr; delimiter: string }
   | { type: 'op' }
   | { type: 'prefix' }