|
| 1 | +// External tokenizer for typescript (generated by gen-lezer.ts). |
| 2 | +// |
| 3 | +// Lezer external tokenizers are plain JS, so we implement the context-sensitive |
| 4 | +// lexing that the grammar's token HINTS describe — the same logic gen-lexer.ts |
| 5 | +// uses — more fully than a Lezer-native C scanner could. Wire the exported |
| 6 | +// tokenizers into the .grammar via: |
| 7 | +// |
| 8 | +// @external tokens contextTokens from "./tokens.js" { Regex, Template, TemplateHead, TemplateMiddle, TemplateTail } |
| 9 | +// |
| 10 | +// This module must also supply these tokens that exceeded Lezer's @tokens syntax |
| 11 | +// (straightforward longest-match scanners — INCOMPLETE, regex listed for porting): |
| 12 | +// JSDoc: /\/\*\*[\s\S]*?\*\// (skip) |
| 13 | +// BlockComment: /\/\*[\s\S]*?\*\// (skip) |
| 14 | +// Ident: /(?:[a-zA-Z_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})(?:[a-zA-Z0-9_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})*/ |
| 15 | +// Number: /[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?/ |
| 16 | +// String: /"(?:[^"\\]|\\[\s\S])*"|'(?:[^'\\]|\\[\s\S])*'/ |
| 17 | +// Decorator: /@(?:[a-zA-Z_$][a-zA-Z0-9_$.]*)?/ |
| 18 | +// |
| 19 | +// NOTE: `@lezer/generator` passes the matching term ids in; the names below must |
| 20 | +// match the @external tokens { ... } list. Replace the term imports accordingly. |
| 21 | + |
| 22 | +import { ExternalTokenizer } from "@lezer/lr"; |
| 23 | +// import { Regex, Template, TemplateHead, TemplateMiddle, TemplateTail } from "./parser.terms.js"; |
| 24 | + |
| 25 | +// Characters/texts after which `/` is DIVISION (value-producing), not a regex. |
| 26 | +const divisionAfterTexts = [")","]","++","--","this","super","true","false","null","undefined"]; |
| 27 | +// Keywords that re-enter expression position, so `/` after them IS a regex. |
| 28 | +const regexAfterTexts = ["in","of","instanceof","typeof","delete","void","await","yield","throw","return","case","do","else","new"]; |
| 29 | + |
| 30 | +// The JS regex for a regex literal (from the grammar's @regex token). |
| 31 | +// Built via RegExp(source) so the pattern's own \/ escapes survive verbatim |
| 32 | +// (embedding in a /.../ literal would double-escape them). 'y' = sticky. |
| 33 | +const REGEX_LITERAL = new RegExp("\\/(?:[^\\/\\\\\\[\\n]|\\\\.|\\[(?:[^\\]\\\\\\n]|\\\\.)*\\])+\\/[gimsuydv]*", "y"); |
| 34 | + |
| 35 | +const IDENT_CHAR = /[\p{L}\p{Nl}\p{Nd}\p{Mn}\p{Mc}\p{Pc}_$]/u; |
| 36 | + |
| 37 | +/** |
| 38 | + * Regex-vs-division tokenizer. Lezer calls this where a `Regex` token is valid. |
| 39 | + * We decide using the PREVIOUS significant token's text/type, mirroring |
| 40 | + * gen-lexer.ts's `divisionPrevTexts`/`expressionStartKeywords` logic. |
| 41 | + * |
| 42 | + * INCOMPLETE: Lezer's `InputStream` does not expose the previous token's TYPE, |
| 43 | + * only characters. We approximate the "division after a value" rule by scanning |
| 44 | + * backwards over whitespace to the previous non-space char and checking the |
| 45 | + * divisionAfterTexts set. Token-TYPE-sensitive cases (Ident/Number/String/…) |
| 46 | + * are handled by Lezer's own tokens taking priority; this hook only fires the |
| 47 | + * regex when a `/` is genuinely in expression position. |
| 48 | + */ |
| 49 | +export const contextTokens = new ExternalTokenizer((input, stack) => { |
| 50 | + const next = input.next; |
| 51 | + if (next !== 47 /* '/' */) return; |
| 52 | + |
| 53 | + // Look back at the previous non-whitespace char already consumed. |
| 54 | + let back = -1; |
| 55 | + let prev = input.peek(back); |
| 56 | + while (prev === 32 || prev === 9 || prev === 10 || prev === 13) { back--; prev = input.peek(back); } |
| 57 | + |
| 58 | + if (prev >= 0) { |
| 59 | + const prevChar = String.fromCharCode(prev); |
| 60 | + // After a closing bracket / identifier char / digit → division, not regex. |
| 61 | + if (prevChar === ")" || prevChar === "]" || prevChar === "}") return; |
| 62 | + if (/[A-Za-z0-9_$]/.test(prevChar)) { |
| 63 | + // Could be an identifier OR an expression-start keyword (return, typeof, …). |
| 64 | + // Read the whole preceding word and consult regexAfterTexts. |
| 65 | + let w = "", b = back; |
| 66 | + let ch = input.peek(b); |
| 67 | + while (ch >= 0 && /[A-Za-z0-9_$]/.test(String.fromCharCode(ch))) { w = String.fromCharCode(ch) + w; b--; ch = input.peek(b); } |
| 68 | + if (!regexAfterTexts.includes(w)) return; // a plain identifier → division |
| 69 | + // else: keyword like `return` → fall through and match a regex literal |
| 70 | + } |
| 71 | + } |
| 72 | + |
| 73 | + // Try to match a regex literal at the current position. |
| 74 | + REGEX_LITERAL.lastIndex = 0; |
| 75 | + let s = ""; |
| 76 | + // Re-read from the input stream char-by-char to feed the sticky regex. |
| 77 | + // (InputStream is forward-only; accumulate then test.) |
| 78 | + let i = 0, c = input.peek(i); |
| 79 | + // Bound the scan to a single line to avoid runaway on malformed input. |
| 80 | + while (c >= 0 && c !== 10 && i < 5000) { s += String.fromCharCode(c); i++; c = input.peek(i); } |
| 81 | + const m = REGEX_LITERAL.exec(s); |
| 82 | + if (m && m.index === 0) { |
| 83 | + input.advance(m[0].length); |
| 84 | + input.acceptToken(/* Regex */ 1); |
| 85 | + } |
| 86 | +}); |
| 87 | + |
| 88 | +// ── Template-literal tokenizer ── |
| 89 | +// Splits an interpolated template into Head / Middle / Tail around `${…}` |
| 90 | +// holes, matching gen-lexer.ts's scanTemplateSpan. Delimiters come from the |
| 91 | +// grammar's template token hint (language-agnostic). |
| 92 | +const TPL_OPEN = "`"; |
| 93 | +const TPL_INTERP_OPEN = "${"; |
| 94 | +const TPL_INTERP_CLOSE = "}"; |
| 95 | + |
| 96 | +/** |
| 97 | + * INCOMPLETE: a production template tokenizer must track interpolation-brace |
| 98 | + * nesting depth across tokenizer invocations (Lezer re-enters per token). The |
| 99 | + * scaffold below tokenizes a non-interpolated template fully and emits a |
| 100 | + * Head token up to the first interpolation; the Middle/Tail transitions need a |
| 101 | + * small stack threaded through the parser's context (`stack`). Mark and finish |
| 102 | + * when wiring into a concrete @lezer/lr build. |
| 103 | + */ |
| 104 | +export const templateTokens = new ExternalTokenizer((input, stack) => { |
| 105 | + let i = 0; |
| 106 | + // expect the opening delimiter |
| 107 | + for (let k = 0; k < TPL_OPEN.length; k++) { |
| 108 | + if (input.peek(i) !== TPL_OPEN.charCodeAt(k)) return; |
| 109 | + i++; |
| 110 | + } |
| 111 | + // scan to closing delimiter or interpolation hole |
| 112 | + while (true) { |
| 113 | + const c = input.peek(i); |
| 114 | + if (c < 0) return; // unterminated |
| 115 | + if (c === 92 /* \\ */) { i += 2; continue; } |
| 116 | + if (startsWith(input, i, TPL_INTERP_OPEN)) { |
| 117 | + input.advance(i + TPL_INTERP_OPEN.length); |
| 118 | + input.acceptToken(/* TemplateHead */ 2); // INCOMPLETE: distinguish Head vs Middle via stack |
| 119 | + return; |
| 120 | + } |
| 121 | + if (startsWith(input, i, TPL_OPEN)) { |
| 122 | + input.advance(i + TPL_OPEN.length); |
| 123 | + input.acceptToken(/* Template */ 3); |
| 124 | + return; |
| 125 | + } |
| 126 | + i++; |
| 127 | + } |
| 128 | +}); |
| 129 | + |
| 130 | +function startsWith(input, at, s) { |
| 131 | + for (let k = 0; k < s.length; k++) { |
| 132 | + if (input.peek(at + k) !== s.charCodeAt(k)) return false; |
| 133 | + } |
| 134 | + return true; |
| 135 | +} |
0 commit comments