|
| 1 | +import type { CstGrammar } from './types.ts'; |
| 2 | +import { collectLiterals, isKeywordLiteral } from './grammar-utils.ts'; |
| 3 | + |
| 4 | +// A lexer token: a declared token (type = its name) or a punctuation literal (type = ''). |
| 5 | +// `$templateHead/$templateMiddle/$templateTail` are synthetic types the lexer emits for |
| 6 | +// the pieces of an interpolated template — role names, not language-specific. |
| 7 | +export interface Token { |
| 8 | + type: string; // token decl name (e.g. 'Ident'), or '' for punctuation literals |
| 9 | + text: string; |
| 10 | + offset: number; |
| 11 | +} |
| 12 | + |
| 13 | +// Build a standalone lexer from the grammar's token definitions + lexer hints. |
| 14 | +// It depends ONLY on tokens/precs — never on the parse rules — so it is the first |
| 15 | +// derived stage: grammar → lexer → parser (and grammar → highlighter), all from one |
| 16 | +// definition. The parser composes this (see gen-parser.ts). |
| 17 | +export function createLexer(grammar: CstGrammar) { |
| 18 | + // Punctuation literals from rules + operators (everything that isn't a keyword word). |
| 19 | + const allLiterals = new Set<string>(); |
| 20 | + for (const rule of grammar.rules) for (const l of collectLiterals(rule.body)) allLiterals.add(l); |
| 21 | + for (const level of grammar.precs) |
| 22 | + for (const op of level.operators) allLiterals.add(op.value); |
| 23 | + const punctLiterals = [...allLiterals] |
| 24 | + .filter(l => !isKeywordLiteral(l)) |
| 25 | + .sort((a, b) => b.length - a.length); |
| 26 | + |
| 27 | + // Token matchers (order matters: earlier declarations win) |
| 28 | + const tokenMatchers = grammar.tokens.map(t => ({ |
| 29 | + name: t.name, |
| 30 | + regex: new RegExp(`^(?:${t.pattern})`), |
| 31 | + skip: t.flags.includes('skip'), |
| 32 | + isRegex: t.flags.includes('regex'), |
| 33 | + })); |
| 34 | + |
| 35 | + // ── Lexer hints (declared per-token in the grammar; nothing here hardcodes a |
| 36 | + // specific language's tokens — see the `identifier`/`template`/`regexContext` opts) ── |
| 37 | + const identTokenName = grammar.tokens.find(t => t.identifier)?.name; |
| 38 | + const templateToken = grammar.tokens.find(t => t.template); |
| 39 | + const templateTokenName = templateToken?.name; |
| 40 | + const tplOpen = templateToken?.template?.open ?? ''; |
| 41 | + const tplInterpOpen = templateToken?.template?.interpOpen ?? ''; |
| 42 | + const tplInterpClose = templateToken?.template?.interpClose ?? ''; |
| 43 | + const tplBraceOpen = tplInterpOpen.slice(-1); // brace that deepens interp nesting ('{' of '${') |
| 44 | + const tplOpenCode = tplOpen.length === 1 ? tplOpen.charCodeAt(0) : -1; // fast path when the open delimiter is one char |
| 45 | + |
| 46 | + // Regex-vs-division context: declared by the grammar's `regex` token. ($templateTail |
| 47 | + // is the lexer's own synthetic template-end token — always a completed value, so `/` |
| 48 | + // after it is division in any language; added here rather than asked of the grammar.) |
| 49 | + const regexCtx = grammar.tokens.find(t => t.regexContext)?.regexContext; |
| 50 | + const divisionPrevTypes = new Set([...(regexCtx?.divisionAfterTypes ?? []), '$templateTail']); |
| 51 | + const divisionPrevTexts = new Set(regexCtx?.divisionAfterTexts ?? []); |
| 52 | + const expressionStartKeywords = new Set(regexCtx?.regexAfterTexts ?? []); |
| 53 | + |
| 54 | + // Scan from inside a template span to its next boundary: an interpolation hole |
| 55 | + // (`interpOpen`) or the closing delimiter (`open`). Delimiters come from the |
| 56 | + // grammar's template token; only called when such a token is declared. |
| 57 | + function scanTemplateSpan(source: string, pos: number): { endsWithInterp: boolean; end: number } { |
| 58 | + while (pos < source.length) { |
| 59 | + if (source[pos] === '\\') { |
| 60 | + pos += 2; |
| 61 | + } else if (source.startsWith(tplInterpOpen, pos)) { |
| 62 | + return { endsWithInterp: true, end: pos + tplInterpOpen.length }; |
| 63 | + } else if (source.startsWith(tplOpen, pos)) { |
| 64 | + return { endsWithInterp: false, end: pos + tplOpen.length }; |
| 65 | + } else { |
| 66 | + pos++; |
| 67 | + } |
| 68 | + } |
| 69 | + throw new Error(`Unterminated template literal at offset ${pos}`); |
| 70 | + } |
| 71 | + |
| 72 | + function tokenize(source: string): Token[] { |
| 73 | + const tokens: Token[] = []; |
| 74 | + let pos = 0; |
| 75 | + const templateStack: number[] = []; |
| 76 | + |
| 77 | + while (pos < source.length) { |
| 78 | + // Skip whitespace |
| 79 | + const wsMatch = source.slice(pos).match(/^\s+/); |
| 80 | + if (wsMatch) { pos += wsMatch[0].length; continue; } |
| 81 | + |
| 82 | + // Close an interpolation hole (interpClose at baseline depth) → resume the template span. |
| 83 | + if (templateStack.length > 0 && source.startsWith(tplInterpClose, pos)) { |
| 84 | + const depth = templateStack[templateStack.length - 1]; |
| 85 | + if (depth === 0) { |
| 86 | + templateStack.pop(); |
| 87 | + const startPos = pos; |
| 88 | + pos += tplInterpClose.length; |
| 89 | + const { endsWithInterp, end } = scanTemplateSpan(source, pos); |
| 90 | + if (endsWithInterp) { |
| 91 | + tokens.push({ type: '$templateMiddle', text: source.slice(startPos, end), offset: startPos }); |
| 92 | + templateStack.push(0); |
| 93 | + } else { |
| 94 | + tokens.push({ type: '$templateTail', text: source.slice(startPos, end), offset: startPos }); |
| 95 | + } |
| 96 | + pos = end; |
| 97 | + continue; |
| 98 | + } else { |
| 99 | + templateStack[templateStack.length - 1]--; |
| 100 | + } |
| 101 | + } |
| 102 | + |
| 103 | + // Track nested opening braces inside an interpolation hole |
| 104 | + if (templateStack.length > 0 && source.startsWith(tplBraceOpen, pos)) { |
| 105 | + templateStack[templateStack.length - 1]++; |
| 106 | + } |
| 107 | + |
| 108 | + // Template literal (simple or interpolated) — only if the grammar declares a template token. |
| 109 | + if (templateToken && (tplOpenCode >= 0 ? source.charCodeAt(pos) === tplOpenCode : source.startsWith(tplOpen, pos))) { |
| 110 | + const startPos = pos; |
| 111 | + pos += tplOpen.length; |
| 112 | + const { endsWithInterp, end } = scanTemplateSpan(source, pos); |
| 113 | + if (endsWithInterp) { |
| 114 | + tokens.push({ type: '$templateHead', text: source.slice(startPos, end), offset: startPos }); |
| 115 | + templateStack.push(0); |
| 116 | + } else { |
| 117 | + tokens.push({ type: templateTokenName!, text: source.slice(startPos, end), offset: startPos }); |
| 118 | + } |
| 119 | + pos = end; |
| 120 | + continue; |
| 121 | + } |
| 122 | + |
| 123 | + const remaining = source.slice(pos); |
| 124 | + let matched = false; |
| 125 | + |
| 126 | + // Try token patterns in declaration order (the template token is handled above) |
| 127 | + for (const tm of tokenMatchers) { |
| 128 | + if (tm.name === templateTokenName) continue; |
| 129 | + if (tm.isRegex) { |
| 130 | + const prev = tokens[tokens.length - 1]; |
| 131 | + if (prev) { |
| 132 | + // Expression-start keywords (in, throw, return, etc.) flip back to regex context |
| 133 | + const isExprKeyword = prev.type === identTokenName && expressionStartKeywords.has(prev.text); |
| 134 | + if (!isExprKeyword && (divisionPrevTypes.has(prev.type) || divisionPrevTexts.has(prev.text))) { |
| 135 | + continue; |
| 136 | + } |
| 137 | + } |
| 138 | + } |
| 139 | + const m = remaining.match(tm.regex); |
| 140 | + if (m) { |
| 141 | + if (!tm.skip) { |
| 142 | + tokens.push({ type: tm.name, text: m[0], offset: pos }); |
| 143 | + } |
| 144 | + pos += m[0].length; |
| 145 | + matched = true; |
| 146 | + break; |
| 147 | + } |
| 148 | + } |
| 149 | + |
| 150 | + if (!matched) { |
| 151 | + // Try punctuation literals (longest first) |
| 152 | + for (const lit of punctLiterals) { |
| 153 | + if (remaining.startsWith(lit)) { |
| 154 | + tokens.push({ type: '', text: lit, offset: pos }); |
| 155 | + pos += lit.length; |
| 156 | + matched = true; |
| 157 | + break; |
| 158 | + } |
| 159 | + } |
| 160 | + } |
| 161 | + |
| 162 | + if (!matched && identTokenName) { |
| 163 | + // Fallback: a Unicode identifier the declared identifier token's pattern may have |
| 164 | + // missed (e.g. accented or non-Latin names). Tagged with that token's name. |
| 165 | + const identMatch = remaining.match(/^[\p{L}\p{Nl}_$][\p{L}\p{Nl}\p{Nd}\p{Mn}\p{Mc}\p{Pc}_$]*/u); |
| 166 | + if (identMatch) { |
| 167 | + tokens.push({ type: identTokenName, text: identMatch[0], offset: pos }); |
| 168 | + pos += identMatch[0].length; |
| 169 | + matched = true; |
| 170 | + } |
| 171 | + } |
| 172 | + |
| 173 | + if (!matched) { |
| 174 | + throw new Error(`Unexpected character at offset ${pos}: '${source[pos]}'`); |
| 175 | + } |
| 176 | + } |
| 177 | + |
| 178 | + return tokens; |
| 179 | + } |
| 180 | + |
| 181 | + return { tokenize }; |
| 182 | +} |
0 commit comments