not()/sameLine + generic/computed grammar

Author: johnsoncodehk

examples/typescript.ts

@@ -1,8 +1,8 @@
 import {
   token, rule, defineGrammar,
   left, right, none,
-  op, prefix, postfix,
-  sep, opt, many, many1, alt,
+  op, prefix, postfix, sameLine,
+  sep, opt, many, many1, alt, not,
 } from '../src/api.ts';
 
 // ── Tokens ──
@@ -70,6 +70,7 @@ const TypeMember = rule($ => {
         [':', Type, ']', opt(':', Type)],                                             // index:  k: T
       )],
       [Expr, ']', opt('?'), propOrMethod],                                            // computed: expr
+      [']', opt(':', Type)],                                                          // empty index sig: []  /  []: T
     )],
     // readonly property (the readonly index signature is the bracketed branch above)
     ['readonly', Ident, opt('?'), ':', Type],
@@ -83,7 +84,7 @@ const Type = rule($ => {
   return [
     [Ident, opt('is', $)],   // T  |  type predicate `x is T`
     [$, '<', sep($, ','), '>'],
-    [$, '[', ']'],
+    [$, sameLine, '[', ']'],   // array type T[] — `[` must be on the same line (no ASI)
     [$, '|', $],
     [$, '&', $],
     ['|', $],   // leading pipe: type T = | A | B
@@ -111,7 +112,7 @@ const Type = rule($ => {
     ['unique', 'symbol'],
     ['import', '(', $, ')'],
     Template,
-    [$, '[', $, ']'],
+    [$, sameLine, '[', $, ']'],   // indexed access T[K] — `[` must be on the same line (no ASI)
     [$, '.', Ident],
   ];
 }, { type: true });
@@ -162,7 +163,10 @@ const Expr = rule($ => [
   [$, postfix],
   ['...', $],
   // instantiation / typed call / tagged template: f<T> | f<T>(…) | f<T>`…`
-  [$, '<', sep(Type, ','), '>', opt(alt(['(', sep($, ','), ')'], Template))],
+  // A bare instantiation `f<T>` (no call/tag) only when the next token can't
+  // start an expression — otherwise `<`/`>` were comparisons (`f < a, b > 7`):
+  // the same disambiguation TS makes via canFollowTypeArgumentsInExpression.
+  [$, '<', sep(Type, ','), '>', alt(['(', sep($, ','), ')'], Template, not(Expr))],
   [$, '(', sep($, ','), ')'],
   [$, '.', alt(Ident, PrivateField)],
   // optional chaining: ?.x | ?.#x | ?.(args) | ?.[i] | ?.`…`

src/api.ts

@@ -61,6 +61,7 @@ export function rule(def: RuleBody, opts?: RuleOptions): RuleRef {
 // ── Special slots for rules ──
 
 interface OpMarker { readonly __kind: 'op' }
+interface SameLineMarker { readonly __kind: 'sameLine' }
 interface PrefixSlot {
   readonly __kind: 'prefix';
   (...ops: string[]): PrefixOps;
@@ -72,10 +73,13 @@ interface PostfixSlot {
 interface PrefixOps { readonly __kind: 'prefix-ops'; ops: string[] }
 interface PostfixOps { readonly __kind: 'postfix-ops'; ops: string[] }
 
-type Marker = OpMarker | PrefixSlot | PostfixSlot;
+type Marker = OpMarker | PrefixSlot | PostfixSlot | SameLineMarker;
 
 export const op: OpMarker = { __kind: 'op' };
 
+// Zero-width "no LineTerminator here" assertion (see RuleExpr 'sameLine').
+export const sameLine: SameLineMarker = { __kind: 'sameLine' };
+
 export const prefix: PrefixSlot = Object.assign(
   (...ops: string[]): PrefixOps => ({ __kind: 'prefix-ops' as const, ops }),
   { __kind: 'prefix' as const },
@@ -119,8 +123,15 @@ class AltNode {
   readonly items: Alternative[];
   constructor(items: Alternative[]) { this.items = items; }
 }
+class NotNode {
+  readonly __kind = 'not' as const;
+  // Zero-width negative lookahead over a single element (wrap a sequence in a
+  // group/alt if needed). Matches nothing; succeeds only when `item` can't match.
+  readonly item: Element;
+  constructor(item: Element) { this.item = item; }
+}
 
-type Combinator = SepNode | OptNode | ManyNode | Many1Node | AltNode;
+type Combinator = SepNode | OptNode | ManyNode | Many1Node | AltNode | NotNode;
 
 export function sep(item: Element, delimiter: string): SepNode {
   return new SepNode(item, delimiter);
@@ -142,6 +153,10 @@ export function alt(...items: Alternative[]): AltNode {
   return new AltNode(items);
 }
 
+export function not(item: Element): NotNode {
+  return new NotNode(item);
+}
+
 // ── Precedence ──
 
 interface PrecLevelDef {
@@ -227,10 +242,14 @@ function toRuleExpr(el: Element, names: Map<object, string>): RuleExpr {
       }),
     };
   }
+  if (el instanceof NotNode) {
+    return { type: 'not', body: toRuleExpr(el.item, names) };
+  }
   const marker = el as Marker;
   if (marker.__kind === 'op') return { type: 'op' };
   if (marker.__kind === 'prefix') return { type: 'prefix' };
   if (marker.__kind === 'postfix') return { type: 'postfix' };
+  if (marker.__kind === 'sameLine') return { type: 'sameLine' };
   throw new Error(`Unknown element: ${JSON.stringify(el)}`);
 }
 

src/cli.ts

@@ -94,6 +94,8 @@ function formatExpr(expr: RuleExpr): string {
     case 'alt': return expr.items.map(formatExpr).join(' | ');
     case 'quantifier': return `${formatExpr(expr.body)}${expr.kind}`;
     case 'group': return `(${formatExpr(expr.body)})`;
+    case 'not': return `not(${formatExpr(expr.body)})`;
+    case 'sameLine': return 'sameLine';
     case 'sep': return `sep(${formatExpr(expr.element)}, '${expr.delimiter}')`;
   }
 }

src/gen-lexer.ts

@@ -8,6 +8,7 @@ export interface Token {
   type: string;   // token decl name (e.g. 'Ident'), or '' for punctuation literals
   text: string;
   offset: number;
+  newlineBefore?: boolean;   // a line terminator preceded this token (drives ASI / "no LineTerminator here" rules)
 }
 
 // Build a standalone lexer from the grammar's token definitions + lexer hints.
@@ -73,11 +74,20 @@ export function createLexer(grammar: CstGrammar) {
     const tokens: Token[] = [];
     let pos = 0;
     const templateStack: number[] = [];
+    // A line terminator was seen since the last emitted token (skipped whitespace
+    // or comments). Stamped onto the next real token as `newlineBefore`, then
+    // cleared — so the parser can honor "no LineTerminator here" restrictions
+    // (e.g. an array/indexed-access type's `[` must be on the same line).
+    let pendingNl = false;
+    function push(t: Token): void {
+      if (pendingNl) { t.newlineBefore = true; pendingNl = false; }
+      tokens.push(t);
+    }
 
     while (pos < source.length) {
       // Skip whitespace
       const wsMatch = source.slice(pos).match(/^\s+/);
-      if (wsMatch) { pos += wsMatch[0].length; continue; }
+      if (wsMatch) { if (wsMatch[0].includes('\n')) pendingNl = true; pos += wsMatch[0].length; continue; }
 
       // Close an interpolation hole (interpClose at baseline depth) → resume the template span.
       if (templateStack.length > 0 && source.startsWith(tplInterpClose, pos)) {
@@ -88,10 +98,10 @@ export function createLexer(grammar: CstGrammar) {
           pos += tplInterpClose.length;
           const { endsWithInterp, end } = scanTemplateSpan(source, pos);
           if (endsWithInterp) {
-            tokens.push({ type: '$templateMiddle', text: source.slice(startPos, end), offset: startPos });
+            push({ type: '$templateMiddle', text: source.slice(startPos, end), offset: startPos });
             templateStack.push(0);
           } else {
-            tokens.push({ type: '$templateTail', text: source.slice(startPos, end), offset: startPos });
+            push({ type: '$templateTail', text: source.slice(startPos, end), offset: startPos });
           }
           pos = end;
           continue;
@@ -111,10 +121,10 @@ export function createLexer(grammar: CstGrammar) {
         pos += tplOpen.length;
         const { endsWithInterp, end } = scanTemplateSpan(source, pos);
         if (endsWithInterp) {
-          tokens.push({ type: '$templateHead', text: source.slice(startPos, end), offset: startPos });
+          push({ type: '$templateHead', text: source.slice(startPos, end), offset: startPos });
           templateStack.push(0);
         } else {
-          tokens.push({ type: templateTokenName!, text: source.slice(startPos, end), offset: startPos });
+          push({ type: templateTokenName!, text: source.slice(startPos, end), offset: startPos });
         }
         pos = end;
         continue;
@@ -139,7 +149,9 @@ export function createLexer(grammar: CstGrammar) {
         const m = remaining.match(tm.regex);
         if (m) {
           if (!tm.skip) {
-            tokens.push({ type: tm.name, text: m[0], offset: pos });
+            push({ type: tm.name, text: m[0], offset: pos });
+          } else if (m[0].includes('\n')) {
+            pendingNl = true;   // a skipped comment spanning a newline still terminates the previous line
           }
           pos += m[0].length;
           matched = true;
@@ -151,7 +163,7 @@ export function createLexer(grammar: CstGrammar) {
         // Try punctuation literals (longest first)
         for (const lit of punctLiterals) {
           if (remaining.startsWith(lit)) {
-            tokens.push({ type: '', text: lit, offset: pos });
+            push({ type: '', text: lit, offset: pos });
             pos += lit.length;
             matched = true;
             break;
@@ -164,7 +176,7 @@ export function createLexer(grammar: CstGrammar) {
         // missed (e.g. accented or non-Latin names). Tagged with that token's name.
         const identMatch = remaining.match(/^[\p{L}\p{Nl}_$][\p{L}\p{Nl}\p{Nd}\p{Mn}\p{Mc}\p{Pc}_$]*/u);
         if (identMatch) {
-          tokens.push({ type: identTokenName, text: identMatch[0], offset: pos });
+          push({ type: identTokenName, text: identMatch[0], offset: pos });
           pos += identMatch[0].length;
           matched = true;
         }

src/gen-parser.ts

@@ -201,6 +201,7 @@ export function createParser(grammar: CstGrammar) {
       case 'alt': return e.items.some(exprNullable);
       case 'quantifier': return e.kind === '+' ? exprNullable(e.body) : true;
       case 'group': return exprNullable(e.body);
+      case 'not': return true;                                   // zero-width assertion: consumes nothing
       case 'sep': return true;                                   // sep matches zero elements
       default: return true;                                      // op/prefix/postfix markers don't consume
     }
@@ -223,7 +224,7 @@ export function createParser(grammar: CstGrammar) {
         const acc = new Set<string>();
         for (const item of e.items) {
           if (item.type === 'prefix') return null;               // prefix op → any operator token: give up
-          if (item.type === 'op' || item.type === 'postfix') continue;  // non-consuming here
+          if (item.type === 'op' || item.type === 'postfix' || item.type === 'not' || item.type === 'sameLine') continue;  // non-consuming here
           const f = exprFirst(item);
           if (f === null) return null;
           for (const k of f) acc.add(k);
@@ -241,6 +242,7 @@ export function createParser(grammar: CstGrammar) {
         return acc;
       }
       case 'quantifier': case 'group': return exprFirst(e.body);
+      case 'not': case 'sameLine': return new Set();             // zero-width: contributes no FIRST tokens
       case 'sep': return exprFirst(e.element);
       default: return null;
     }
@@ -626,6 +628,21 @@ export function createParser(grammar: CstGrammar) {
           return matchQuantifier(expr.body, expr.kind);
         case 'group':
           return matchExpr(expr.body);
+        case 'not': {
+          // Zero-width negative lookahead: succeed (no children) iff the body
+          // does NOT match here; never consume input either way.
+          const saved = pos;
+          const m = matchExpr(expr.body);
+          pos = saved;
+          return m === null ? [] : null;
+        }
+        case 'sameLine': {
+          // Zero-width "no LineTerminator here": succeed (no children) iff the
+          // next token isn't preceded by a newline. At EOF there's no token to
+          // continue onto, so the assertion fails (nothing same-line follows).
+          const tok = peek();
+          return tok && !tok.newlineBefore ? [] : null;
+        }
         case 'sep':
           return matchSep(expr.element, expr.delimiter);
         default:

src/gen-treesitter.ts

@@ -139,6 +139,16 @@ function renderExpr(expr: RuleExpr, ctx: GrammarJsContext): string {
     }
     case 'group':
       return renderExpr(expr.body, ctx);
+    case 'not':
+      // Zero-width negative lookahead: not expressible in a tree-sitter CFG, and
+      // it consumes nothing, so it drops to a no-op (the surrounding choice keeps
+      // the bare form, as the grammar did before the assertion was added).
+      return 'blank()';
+    case 'sameLine':
+      // Zero-width "no LineTerminator here" assertion — tree-sitter handles this
+      // class of restriction with an external scanner, not the CFG; as a CFG node
+      // it consumes nothing, so render a no-op.
+      return 'blank()';
     case 'sep': {
       // sep(elem, ',') = optional(seq(elem, repeat(seq(',', elem)), optional(',')))
       // Trailing delimiter is allowed (matches the parser's matchSep behavior).

src/types.ts

@@ -44,6 +44,19 @@ export type RuleExpr =
   | { type: 'ref'; name: string }
   | { type: 'quantifier'; body: RuleExpr; kind: '*' | '+' | '?' }
   | { type: 'group'; body: RuleExpr }
+  // Zero-width negative lookahead: matches (consuming nothing) iff `body` does
+  // NOT match at the current position. Used to express disambiguations the
+  // longest-match parser can't reach by structure alone (e.g. a `<…>` type-arg
+  // list in expression position is only a bare instantiation when it isn't
+  // followed by something that starts an expression). Non-consuming → invisible
+  // to highlighting / AST shape / other generators.
+  | { type: 'not'; body: RuleExpr }
+  // Zero-width "no LineTerminator here" assertion: matches (consuming nothing)
+  // iff the NEXT token is on the same line (no preceding newline). Encodes
+  // ECMAScript/TS restricted productions like an array/indexed-access type's `[`,
+  // which must not follow a line terminator. Non-consuming → invisible to other
+  // generators (they treat it as a no-op marker).
+  | { type: 'sameLine' }
   | { type: 'sep'; element: RuleExpr; delimiter: string }
   | { type: 'op' }
   | { type: 'prefix' }