Make token-pattern literals bare strings (#16)

A token-pattern literal is now a bare `string` instead of a `lit('x')`
wrapper. This removes the entire layer that existed only to box a string
into a TokenPattern: `lit`, `TokenPatternInput`, `toTokenPattern` /
`toPatternInput`, `isTokenPattern`, `TokenCharClassInput`, `mark` + the
`__kind: 'token-pattern'` marker, and `NormalizedTokenOptions` /
`normalizeTokenOptions`.

Because a bare string now inhabits both the token-pattern world (a
literal) and the rule world (a token reference), the first-argument
overload that `alt`/`opt` relied on can no longer dispatch. They are
split into token-pattern combinators `altPattern`/`optPattern` and the
rule combinators `alt`/`opt`. Grammars, tests, and README examples are
updated accordingly.

All 7 generated grammars are byte-identical; conformance unchanged
(5386/5659).

Author: johnsoncodehk

README.md

@@ -242,15 +242,15 @@ A grammar is a TypeScript module: tokens, operator precedence, and rules built f
 ```ts
 import {
   token, rule, defineGrammar, left, op, sep,
-  seq, oneOf, range, plus, star, opt,
+  seq, oneOf, range, plus, star, optPattern,
 } from './src/api.ts';
 
 const digit = range('0', '9');
 const Ident = token(seq(
   oneOf(range('a', 'z'), range('A', 'Z'), '_', '$'),
   star(oneOf(range('a', 'z'), range('A', 'Z'), digit, '_', '$')),
 ), { identifier: true });
-const Number = token(seq(plus(digit), opt(seq('.', plus(digit)))));
+const Number = token(seq(plus(digit), optPattern(seq('.', plus(digit)))));
 
 const Expr = rule($ => [
   Ident,
@@ -283,23 +283,23 @@ Flat, irreducible facts — which keywords are control flow, which punctuation i
 Nothing in the engine knows about TypeScript. Everything language-specific lives in the grammar — keywords, which token is the identifier, template-literal delimiters, the regex-vs-division lexer ambiguity — all *declared per token*:
 
 ```ts
-import { token, seq, alt, noneOf, anyChar, oneOf, plus, star, notFollowedBy } from './src/api.ts';
+import { token, seq, altPattern, noneOf, anyChar, oneOf, plus, star, notFollowedBy } from './src/api.ts';
 
 const escaped = seq('\\', anyChar());
 
 const Template = token(seq(
   '`',
-  star(alt(noneOf('`', '\\', '$'), escaped, seq('$', notFollowedBy('{')))),
+  star(altPattern(noneOf('`', '\\', '$'), escaped, seq('$', notFollowedBy('{')))),
   '`',
 ), {
   template: { open: '`', interpOpen: '${', interpClose: '}' },
 });
 const Regex = token(seq(
   '/',
-  plus(alt(
+  plus(altPattern(
     noneOf('/', '\\', '[', '\n'),
     escaped,
-    seq('[', star(alt(noneOf(']', '\\', '\n'), escaped)), ']'),
+    seq('[', star(altPattern(noneOf(']', '\\', '\n'), escaped)), ']'),
   )),
   '/',
   star(oneOf('g', 'i', 'm', 's', 'u', 'y', 'd', 'v')),

html.ts

@@ -9,7 +9,7 @@
 // tags). It does not implement the WHATWG error-recovery tree-construction
 // algorithm (that is not a context-free grammar); conformance is measured against
 // `parse5` on well-formed input. See memory: html-vue-markup.
-import { token, rule, defineGrammar, many, opt, alt, lit, seq, oneOf, noneOf, range, anyChar, star, plus, notFollowedBy } from './src/api.ts';
+import { token, rule, defineGrammar, many, opt, alt, altPattern, seq, oneOf, noneOf, range, anyChar, star, plus, notFollowedBy } from './src/api.ts';
 import type { MarkupConfig } from './src/types.ts';
 
 // ── Tokens ──
@@ -27,7 +27,7 @@ const Name = token(seq(oneOf(range('a', 'z'), range('A', 'Z')), star(oneOf(word,
 // the generic engine knowing any tag names.
 const VoidName = token(seq(oneOf(range('a', 'z'), range('A', 'Z')), star(oneOf(word, ':', '.', '-'))), { scope: 'entity.name.tag' });
 // Quoted attribute value (double or single).
-const AttrValue = token(alt(seq('"', star(noneOf('"')), '"'), seq("'", star(noneOf("'")), "'")), { string: true });
+const AttrValue = token(altPattern(seq('"', star(noneOf('"')), '"'), seq("'", star(noneOf("'")), "'")), { string: true });
 // Unquoted attribute value (`colspan=2`, `value=5px`, `href=https://x/`, `href=/a/b.css`):
 // per WHATWG, an unquoted value ends ONLY at whitespace or `>`, so `/` is a legal value char
 // (URLs / paths). The lexer scans the whole value as ONE token the moment it follows `=` (see
@@ -122,8 +122,8 @@ export const markup: MarkupConfig = {
   // VS Code's own HTML grammar (a flat source.css blob there) and matching the hand-written Vue
   // grammar's `#vue-directives-style-attr` (its "Copy from source.css#rule-list-innards").
   attributeEmbed: [
-    { namePattern: seq(lit('on'), plus(word)), embed: 'source.js' },
-    { namePattern: lit('style'), embed: 'source.css', include: 'source.css#rule-list-innards' },
+    { namePattern: seq('on', plus(word)), embed: 'source.js' },
+    { namePattern: 'style', embed: 'source.css', include: 'source.css#rule-list-innards' },
   ],
   // Raw-text element bodies are scanned verbatim by the parser, but the HIGHLIGHTER
   // delegates `<script>`/`<style>` to the platform's real JS/CSS grammars (exactly as

javascript.ts

@@ -29,7 +29,7 @@ import {
   left, right, none, noUnaryLhs,
   op, prefix, postfix, sameLine,
   sep, opt, many, many1, alt, exclude, not,
-  lit, seq, oneOf, noneOf, range, anyChar, star, plus, repeat, notFollowedBy, start,
+  altPattern, optPattern, seq, oneOf, noneOf, range, anyChar, star, plus, repeat, notFollowedBy, start,
 } from './src/api.ts';
 
 // ── Tokens ──
@@ -46,9 +46,9 @@ const idStart = oneOf(range('a', 'z'), range('A', 'Z'), '_', '$');
 const idCont = oneOf(range('a', 'z'), range('A', 'Z'), digit, '_', '$');
 const lineTerminator = oneOf('\n', '\r', '\u2028', '\u2029');
 const hspace = oneOf(' ', '\t');
-const uEsc = alt(seq('\\u', repeat(hexDigit, 4, 4)), seq('\\u{', plus(hexDigit), '}'));
-const identStart = alt(idStart, uEsc);
-const identPart = alt(idCont, uEsc);
+const uEsc = altPattern(seq('\\u', repeat(hexDigit, 4, 4)), seq('\\u{', plus(hexDigit), '}'));
+const identStart = altPattern(idStart, uEsc);
+const identPart = altPattern(idCont, uEsc);
 const numericTailGuard = notFollowedBy(oneOf(range('0', '9'), range('A', 'Z'), range('a', 'z'), '_', '$', '\\'));
 const digits = seq(plus(digit), star(seq('_', plus(digit))));
 const Ident        = token(seq(identStart, star(identPart)), { identifier: true });
@@ -63,39 +63,39 @@ const Ident        = token(seq(identStart, star(identPart)), { identifier: true
 // only valid here (radix forms have no fractional part), so it lives on each radix token rather
 // than on the decimal `BigInt_` below. The shared `(?!IdentifierStart|DecimalDigit)` tail still
 // rejects a stray trailing identifier (`0x5anabc`).
-const HexNumber    = token(seq('0', oneOf('x', 'X'), plus(hexDigit), star(seq('_', plus(hexDigit))), opt(lit('n')), numericTailGuard), { scope: 'constant.numeric.hex' });
-const OctalNumber  = token(seq('0', oneOf('o', 'O'), plus(range('0', '7')), star(seq('_', plus(range('0', '7')))), opt(lit('n')), numericTailGuard), { scope: 'constant.numeric.octal' });
-const BinaryNumber = token(seq('0', oneOf('b', 'B'), plus(oneOf('0', '1')), star(seq('_', plus(oneOf('0', '1')))), opt(lit('n')), numericTailGuard), { scope: 'constant.numeric.binary' });
+const HexNumber    = token(seq('0', oneOf('x', 'X'), plus(hexDigit), star(seq('_', plus(hexDigit))), optPattern('n'), numericTailGuard), { scope: 'constant.numeric.hex' });
+const OctalNumber  = token(seq('0', oneOf('o', 'O'), plus(range('0', '7')), star(seq('_', plus(range('0', '7')))), optPattern('n'), numericTailGuard), { scope: 'constant.numeric.octal' });
+const BinaryNumber = token(seq('0', oneOf('b', 'B'), plus(oneOf('0', '1')), star(seq('_', plus(oneOf('0', '1')))), optPattern('n'), numericTailGuard), { scope: 'constant.numeric.binary' });
 const BigInt_      = token(seq(digits, 'n', numericTailGuard), { scope: 'constant.numeric.bigint' });
 // DecimalLiteral, including the leading-dot form (`.5`, `.0e1`): an integer part with optional
 // fraction/exponent, OR a bare fraction `.digits` with optional exponent. Same trailing guard.
 // Scope is set explicitly (not inferred from a `[0-9]`-leading pattern) because the leading-dot
 // alternative makes the pattern start with `(?:` — gen-tm's decimal-numeric detector keys on a
 // `[0-9]`/`\d` prefix, so without this the token would lose its `constant.numeric` scope.
 const fracTail = seq('.', star(digit), star(seq('_', plus(digit))));
-const expTail = seq(oneOf('e', 'E'), opt(oneOf('+', '-')), digits);
-const Number_      = token(seq(alt(seq(digits, opt(fracTail)), seq('.', digits)), opt(expTail), numericTailGuard), { scope: 'constant.numeric.decimal' });
+const expTail = seq(oneOf('e', 'E'), optPattern(oneOf('+', '-')), digits);
+const Number_      = token(seq(altPattern(seq(digits, optPattern(fracTail)), seq('.', digits)), optPattern(expTail), numericTailGuard), { scope: 'constant.numeric.decimal' });
 // A well-formed JS escape, used in the string-body pattern below. `\u`/`\x` must
 // match their strict forms — a `\u{cp}` with cp ≤ 0x10FFFF, a 4-hex `\uXXXX`, or a
 // 2-hex `\xXX` — while `\` + any *other* char (\n, \\, \q non-escape, line
 // continuation) stays valid via `[^ux]`. A malformed `\u`/`\x` (e.g. `\u{110000}`,
 // `\u{r}`, `\u{}`, `\u{67`) matches no escape, so the string matches no token and the
 // lexer throws — TS's exact rejection. The in-range codepoint is `0*` leading zeros
 // then 1–5 hex (0–0xFFFFF) or `10`+4 hex (0x100000–0x10FFFF).
-const codePoint = seq(star('0'), alt(repeat(hexDigit, 1, 5), seq('10', repeat(hexDigit, 4, 4))));
-const escape = seq('\\', alt(seq('u{', codePoint, '}'), seq('u', repeat(hexDigit, 4, 4)), seq('x', repeat(hexDigit, 2, 2)), noneOf('u', 'x')));
-const highlightedEscape = seq('\\', alt(
+const codePoint = seq(star('0'), altPattern(repeat(hexDigit, 1, 5), seq('10', repeat(hexDigit, 4, 4))));
+const escape = seq('\\', altPattern(seq('u{', codePoint, '}'), seq('u', repeat(hexDigit, 4, 4)), seq('x', repeat(hexDigit, 2, 2)), noneOf('u', 'x')));
+const highlightedEscape = seq('\\', altPattern(
   oneOf('n', 'r', 't', 'b', 'f', 'v', '0', "'", '"', '\\'),
   seq('x', repeat(hexDigit, 2, 2)),
   seq('u', repeat(hexDigit, 4, 4)),
   seq('u{', plus(hexDigit), '}'),
 ));
-const String_      = token(alt(seq('"', star(alt(noneOf('"', '\\'), escape)), '"'), seq("'", star(alt(noneOf("'", '\\'), escape)), "'")), {
+const String_      = token(altPattern(seq('"', star(altPattern(noneOf('"', '\\'), escape)), '"'), seq("'", star(altPattern(noneOf("'", '\\'), escape)), "'")), {
   string: true,
   escape: highlightedEscape,
 });
-const Template     = token(seq('`', star(alt(noneOf('`', '\\', '$'), seq('\\', noneOf(lineTerminator)), seq('$', notFollowedBy('{')))), '`'), {
-  escape: seq('\\', alt(
+const Template     = token(seq('`', star(altPattern(noneOf('`', '\\', '$'), seq('\\', noneOf(lineTerminator)), seq('$', notFollowedBy('{')))), '`'), {
+  escape: seq('\\', altPattern(
     oneOf('n', 'r', 't', 'b', 'f', 'v', '0', "'", '"', '\\', '`', '$'),
     seq('x', repeat(hexDigit, 2, 2)),
     seq('u', repeat(hexDigit, 4, 4)),
@@ -107,8 +107,8 @@ const Template     = token(seq('`', star(alt(noneOf('`', '\\', '$'), seq('\\', n
   template: { open: '`', interpOpen: '${', interpClose: '}' },
 });
 const regexEscape = seq('\\', noneOf(lineTerminator));
-const regexClassBody = star(alt(noneOf(']', '\\', '\n'), regexEscape));
-const Regex_       = token(seq('/', plus(alt(noneOf('/', '\\', '[', '\n'), regexEscape, seq('[', regexClassBody, ']'))), '/', star(oneOf('g', 'i', 'm', 's', 'u', 'y', 'd', 'v'))), {
+const regexClassBody = star(altPattern(noneOf(']', '\\', '\n'), regexEscape));
+const Regex_       = token(seq('/', plus(altPattern(noneOf('/', '\\', '[', '\n'), regexEscape, seq('[', regexClassBody, ']'))), '/', star(oneOf('g', 'i', 'm', 's', 'u', 'y', 'd', 'v'))), {
   regex: true,
   regexContext: {
     divisionAfterTypes: ['Ident', 'Number', 'String', 'Template', 'BigInt'],
@@ -131,7 +131,7 @@ const Regex_       = token(seq('/', plus(alt(noneOf('/', '\\', '[', '\n'), regex
 });
 // `@name` / `@ns.name` — each dotted segment is an IdentifierName, so it admits the same
 // `\u`-escape forms as `Ident` (`@℘`, `@ZW_‌_NJ`); the parser owns the `(args)` tail.
-const Decorator    = token(seq('@', opt(seq(identStart, star(alt(identPart, '.'))))), { scope: 'entity.name.function.decorator' });
+const Decorator    = token(seq('@', optPattern(seq(identStart, star(altPattern(identPart, '.'))))), { scope: 'entity.name.function.decorator' });
 // PrivateIdentifier: `#` + an IdentifierName, so it admits the same `\u`-escape forms as `Ident`
 // (`#\u{6F}_`). A non-ASCII literal `#name` (`#℘`, `#ZWNJ`) is handled by the lexer's Unicode
 // fallback, which recognises this token's leading `#` as a name prefix.

src/api.ts

@@ -1,30 +1,29 @@
 import type { CstGrammar, TokenDecl, PrecLevel, PrecOperator, RuleDecl, RuleExpr, MarkupConfig, IndentConfig, NewlineConfig, StringInterpolation, TokenPattern } from './types.ts';
 import {
-  altPattern, anyChar, followedBy, isTokenPattern, lit, never, noneOf, notFollowedBy,
+  altPattern, anyChar, followedBy, never, noneOf, notFollowedBy,
   notPrecededBy, oneOf, optPattern, plus, precededBy, range, repeat,
-  seq, star, start, end, toTokenPattern,
-  type TokenPatternInput,
+  seq, star, start, end,
 } from './token-pattern.ts';
 
 export {
-  anyChar, followedBy, lit, never, noneOf, notFollowedBy, notPrecededBy, oneOf,
-  plus, precededBy, range, repeat, seq, star, start, end,
+  altPattern, anyChar, followedBy, never, noneOf, notFollowedBy, notPrecededBy, oneOf,
+  optPattern, plus, precededBy, range, repeat, seq, star, start, end,
 };
 
 // ── Token ──
 
 interface TokenOptions {
   skip?: boolean;
   scope?: string;
-  escape?: TokenPatternInput;
+  escape?: TokenPattern;
   // Highlight-only interpolation regions for ordinary string tokens (e.g. env-spec `${…}` / `$(…)`).
   // The parser/lexer stay token-based; generators re-express these as nested regions.
   interpolation?: StringInterpolation | StringInterpolation[];
   // A regex matching exactly one well-formed escape sequence. Engine-scanned tokens
   // (templates) validate each `\`-escape against it and reject any that don't match —
   // unlike `escape` (highlight-only), this drives tokenization. Skipped in tag
   // position, where invalid escapes are legal (cooked = undefined). Optional.
-  escapeValid?: TokenPatternInput;
+  escapeValid?: TokenPattern;
   regex?: boolean;
   embed?: string;
   // ── Lexer hints (keep gen-parser language-agnostic; all optional) ──
@@ -45,38 +44,23 @@ interface TokenOptions {
   };
   string?: boolean;
   // Block-context (flowDepth===0) pattern variant for indentation grammars — see TokenDecl.blockPattern.
-  blockPattern?: TokenPatternInput;
+  blockPattern?: TokenPattern;
   // Block-context ONLY (indentation grammars): match this token only outside flow — see TokenDecl.blockOnly.
   blockOnly?: boolean;
 }
 
-type NormalizedTokenOptions = Omit<TokenOptions, 'escape' | 'escapeValid' | 'blockPattern'> & {
-  escape?: TokenPattern;
-  escapeValid?: TokenPattern;
-  blockPattern?: TokenPattern;
-};
-
 export class TokenRef {
   readonly __kind = 'token' as const;
   readonly pattern: TokenPattern;
-  readonly opts: NormalizedTokenOptions;
-  constructor(pattern: TokenPattern, opts: NormalizedTokenOptions) {
+  readonly opts: TokenOptions;
+  constructor(pattern: TokenPattern, opts: TokenOptions) {
     this.pattern = pattern;
     this.opts = opts;
   }
 }
 
-export function token(pattern: TokenPatternInput, opts?: TokenOptions): TokenRef {
-  return new TokenRef(toTokenPattern(pattern), normalizeTokenOptions(opts ?? {}));
-}
-
-function normalizeTokenOptions(opts: TokenOptions): NormalizedTokenOptions {
-  return {
-    ...opts,
-    escape: opts.escape ? toTokenPattern(opts.escape) : undefined,
-    escapeValid: opts.escapeValid ? toTokenPattern(opts.escapeValid) : undefined,
-    blockPattern: opts.blockPattern ? toTokenPattern(opts.blockPattern) : undefined,
-  };
+export function token(pattern: TokenPattern, opts?: TokenOptions): TokenRef {
+  return new TokenRef(pattern, opts ?? {});
 }
 
 // ── Rule ──
@@ -215,11 +199,8 @@ export function sep(item: Element, delimiter: string): SepNode {
   return new SepNode(item, delimiter);
 }
 
-export function opt(...items: [TokenPattern, ...TokenPatternInput[]]): TokenPattern;
-export function opt(...items: Element[]): OptNode;
-export function opt(...items: (Element | TokenPatternInput)[]): OptNode | TokenPattern {
-  if (items.some(isTokenPattern)) return optPattern(...items as TokenPatternInput[]);
-  return new OptNode(items as Element[]);
+export function opt(...items: Element[]): OptNode {
+  return new OptNode(items);
 }
 
 export function many(...items: Element[]): ManyNode {
@@ -230,11 +211,8 @@ export function many1(...items: Element[]): Many1Node {
   return new Many1Node(items);
 }
 
-export function alt(...items: [TokenPattern, ...TokenPatternInput[]]): TokenPattern;
-export function alt(...items: Alternative[]): AltNode;
-export function alt(...items: (Alternative | TokenPatternInput)[]): AltNode | TokenPattern {
-  if (items.some(isTokenPattern)) return altPattern(...items as TokenPatternInput[]);
-  return new AltNode(items as Alternative[]);
+export function alt(...items: Alternative[]): AltNode {
+  return new AltNode(items);
 }
 
 // Parse `items` with the given LED connector(s) disabled at the top level (a