Skip to content

Commit 09f7ed6

Browse files
CopilotSander-Toonen
Copilot
and
Sander-Toonen
committed
Refactor token-stream.ts to reduce cognitive complexity
Co-authored-by: Sander-Toonen <5106372+Sander-Toonen@users.noreply.github.com>
1 parent 0839571 commit 09f7ed6

2 files changed

Lines changed: 208 additions & 123 deletions

File tree

‎src/parsing/token-stream.ts‎

Lines changed: 196 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,25 @@ interface Coordinates {
3030
column: number;
3131
}
3232

33+
/** Single-character operators that map directly to themselves */
34+
const SINGLE_CHAR_OPERATORS = new Set(['+', '-', '*', '/', '%', '^', ':', '.']);
35+
36+
/** Unicode multiplication symbols that map to '*' */
37+
const MULTIPLICATION_SYMBOLS = new Set(['∙', '•']);
38+
39+
/** Escape sequence mappings for string unescape */
40+
const ESCAPE_SEQUENCES: Record<string, string> = {
41+
'\'': '\'',
42+
'"': '"',
43+
'\\': '\\',
44+
'/': '/',
45+
'b': '\b',
46+
'f': '\f',
47+
'n': '\n',
48+
'r': '\r',
49+
't': '\t'
50+
};
51+
3352
export class TokenStream {
3453
public pos: number = 0;
3554
public keywords: string[];
@@ -269,6 +288,31 @@ export class TokenStream {
269288

270289
private static readonly codePointPattern = /^[0-9a-f]{4}$/i;
271290

291+
/**
292+
* Process a single escape sequence character and return the unescaped result
293+
*/
294+
private processEscapeChar(c: string, v: string, currentIndex: number): { char: string; skip: number } {
295+
// Check standard escape sequences first
296+
if (c in ESCAPE_SEQUENCES) {
297+
return { char: ESCAPE_SEQUENCES[c], skip: 0 };
298+
}
299+
300+
// Handle unicode escape sequence
301+
if (c === 'u') {
302+
const codePoint = v.substring(currentIndex + 1, currentIndex + 5);
303+
if (!TokenStream.codePointPattern.test(codePoint)) {
304+
this.parseError('Illegal escape sequence: \\u' + codePoint);
305+
}
306+
return { char: String.fromCharCode(parseInt(codePoint, 16)), skip: 4 };
307+
}
308+
309+
// Unknown escape sequence
310+
throw this.parseError('Illegal escape sequence: "\\' + c + '"');
311+
}
312+
313+
/**
314+
* Unescape a string by processing escape sequences
315+
*/
272316
unescape(v: string): string {
273317
const index = v.indexOf('\\');
274318
if (index < 0) {
@@ -277,48 +321,13 @@ export class TokenStream {
277321

278322
let buffer = v.substring(0, index);
279323
let currentIndex = index;
324+
280325
while (currentIndex >= 0) {
281326
const c = v.charAt(++currentIndex);
282-
switch (c) {
283-
case '\'':
284-
buffer += '\'';
285-
break;
286-
case '"':
287-
buffer += '"';
288-
break;
289-
case '\\':
290-
buffer += '\\';
291-
break;
292-
case '/':
293-
buffer += '/';
294-
break;
295-
case 'b':
296-
buffer += '\b';
297-
break;
298-
case 'f':
299-
buffer += '\f';
300-
break;
301-
case 'n':
302-
buffer += '\n';
303-
break;
304-
case 'r':
305-
buffer += '\r';
306-
break;
307-
case 't':
308-
buffer += '\t';
309-
break;
310-
case 'u':
311-
// interpret the following 4 characters as the hex of the unicode code point
312-
const codePoint = v.substring(currentIndex + 1, currentIndex + 5);
313-
if (!TokenStream.codePointPattern.test(codePoint)) {
314-
this.parseError('Illegal escape sequence: \\u' + codePoint);
315-
}
316-
buffer += String.fromCharCode(parseInt(codePoint, 16));
317-
currentIndex += 4;
318-
break;
319-
default:
320-
throw this.parseError('Illegal escape sequence: "\\' + c + '"');
321-
}
327+
const { char, skip } = this.processEscapeChar(c, v, currentIndex);
328+
buffer += char;
329+
currentIndex += skip;
330+
322331
++currentIndex;
323332
const backslash = v.indexOf('\\', currentIndex);
324333
buffer += v.substring(currentIndex, backslash < 0 ? v.length : backslash);
@@ -441,88 +450,164 @@ export class TokenStream {
441450
return valid;
442451
}
443452

444-
isOperator(): boolean {
445-
const startPos = this.pos;
446-
const c = this.expression.charAt(this.pos);
447-
448-
if (c === '+' || c === '-' || c === '*' || c === '/' || c === '%' || c === '^' || c === ':' || c === '.') {
453+
/**
454+
* Try to match a single-character operator (+, -, *, /, %, ^, :, .)
455+
*/
456+
private tryMatchSingleCharOperator(c: string): boolean {
457+
if (SINGLE_CHAR_OPERATORS.has(c)) {
449458
this.current = this.newToken(TOP, c);
450-
} else if (c === '?') {
451-
// ? could be a ternary operator a ? b : c or a coalesce operator a ?? b, we need to look ahead
452-
// to figure out which one it is.
453-
if (this.expression.charAt(this.pos + 1) === '?') {
454-
if (this.isOperatorEnabled('??')) {
455-
this.current = this.newToken(TOP, '??');
456-
this.pos++;
457-
} else {
458-
// We have a ?? operator but it has been disabled.
459-
return false;
460-
}
461-
} else {
462-
this.current = this.newToken(TOP, c);
463-
}
464-
} else if (c === '∙' || c === '•') {
459+
return true;
460+
}
461+
return false;
462+
}
463+
464+
/**
465+
* Try to match unicode multiplication symbols (∙, •)
466+
*/
467+
private tryMatchMultiplicationSymbol(c: string): boolean {
468+
if (MULTIPLICATION_SYMBOLS.has(c)) {
465469
this.current = this.newToken(TOP, '*');
466-
} else if (c === '>') {
467-
if (this.expression.charAt(this.pos + 1) === '=') {
468-
this.current = this.newToken(TOP, '>=');
469-
this.pos++;
470-
} else {
471-
this.current = this.newToken(TOP, '>');
472-
}
473-
} else if (c === '<') {
474-
if (this.expression.charAt(this.pos + 1) === '=') {
475-
this.current = this.newToken(TOP, '<=');
476-
this.pos++;
477-
} else {
478-
this.current = this.newToken(TOP, '<');
479-
}
480-
} else if (c === '|') {
481-
if (this.expression.charAt(this.pos + 1) === '|') {
482-
this.current = this.newToken(TOP, '||');
483-
this.pos++;
484-
} else {
485-
this.current = this.newToken(TOP, '|');
486-
}
487-
} else if (c === '&') {
488-
if (this.expression.charAt(this.pos + 1) === '&') {
489-
this.current = this.newToken(TOP, '&&');
490-
this.pos++;
491-
} else {
492-
return false;
493-
}
494-
} else if (c === '=') {
495-
if (this.expression.charAt(this.pos + 1) === '=') {
496-
this.current = this.newToken(TOP, '==');
497-
this.pos++;
498-
} else {
499-
this.current = this.newToken(TOP, c);
500-
}
501-
} else if (c === '!') {
502-
if (this.expression.charAt(this.pos + 1) === '=') {
503-
this.current = this.newToken(TOP, '!=');
504-
this.pos++;
505-
} else {
506-
this.current = this.newToken(TOP, c);
507-
}
508-
} else if (c === 'a' && this.expression.substring(this.pos, this.pos + 3) === 'as ') {
509-
if (this.isOperatorEnabled('as')) {
510-
this.current = this.newToken(TOP, 'as');
511-
this.pos++;
512-
} else {
470+
return true;
471+
}
472+
return false;
473+
}
474+
475+
/**
476+
* Try to match the question mark operator (? or ??)
477+
* Returns null if no match, false if disabled, true if matched
478+
*/
479+
private tryMatchQuestionOperator(c: string): boolean | null {
480+
if (c !== '?') {
481+
return null;
482+
}
483+
484+
if (this.expression.charAt(this.pos + 1) === '?') {
485+
if (!this.isOperatorEnabled('??')) {
513486
return false;
514487
}
488+
this.current = this.newToken(TOP, '??');
489+
this.pos++;
490+
} else {
491+
this.current = this.newToken(TOP, c);
492+
}
493+
return true;
494+
}
495+
496+
/**
497+
* Try to match a two-character operator where the second char may be '='
498+
* Examples: >=, <=, ==, !=
499+
*/
500+
private tryMatchComparisonOperator(c: string, doubleOp: string): boolean {
501+
if (this.expression.charAt(this.pos + 1) === '=') {
502+
this.current = this.newToken(TOP, doubleOp);
503+
this.pos++;
504+
} else {
505+
this.current = this.newToken(TOP, c);
506+
}
507+
return true;
508+
}
509+
510+
/**
511+
* Try to match pipe operators (| or ||)
512+
*/
513+
private tryMatchPipeOperator(): boolean {
514+
if (this.expression.charAt(this.pos + 1) === '|') {
515+
this.current = this.newToken(TOP, '||');
516+
this.pos++;
515517
} else {
518+
this.current = this.newToken(TOP, '|');
519+
}
520+
return true;
521+
}
522+
523+
/**
524+
* Try to match the && operator (single & is not valid)
525+
*/
526+
private tryMatchAmpersandOperator(): boolean | null {
527+
if (this.expression.charAt(this.pos + 1) === '&') {
528+
this.current = this.newToken(TOP, '&&');
529+
this.pos++;
530+
return true;
531+
}
532+
return false;
533+
}
534+
535+
/**
536+
* Try to match the 'as' keyword operator
537+
*/
538+
private tryMatchAsOperator(): boolean | null {
539+
if (this.expression.substring(this.pos, this.pos + 3) !== 'as ') {
540+
return null;
541+
}
542+
if (!this.isOperatorEnabled('as')) {
516543
return false;
517544
}
545+
this.current = this.newToken(TOP, 'as');
518546
this.pos++;
547+
return true;
548+
}
519549

520-
if (this.isOperatorEnabled(this.current.value as string)) {
521-
return true;
522-
} else {
523-
this.pos = startPos;
550+
/**
551+
* Attempt to identify the current character as an operator
552+
*/
553+
isOperator(): boolean {
554+
const startPos = this.pos;
555+
const c = this.expression.charAt(this.pos);
556+
557+
// Try single-character operators
558+
if (this.tryMatchSingleCharOperator(c)) {
559+
// matched
560+
}
561+
// Try unicode multiplication symbols
562+
else if (this.tryMatchMultiplicationSymbol(c)) {
563+
// matched
564+
}
565+
// Try question mark operators (? and ??)
566+
else if (c === '?') {
567+
const result = this.tryMatchQuestionOperator(c);
568+
if (result === false) return false;
569+
}
570+
// Try comparison operators with optional '='
571+
else if (c === '>') {
572+
this.tryMatchComparisonOperator(c, '>=');
573+
}
574+
else if (c === '<') {
575+
this.tryMatchComparisonOperator(c, '<=');
576+
}
577+
else if (c === '=') {
578+
this.tryMatchComparisonOperator(c, '==');
579+
}
580+
else if (c === '!') {
581+
this.tryMatchComparisonOperator(c, '!=');
582+
}
583+
// Try pipe operators
584+
else if (c === '|') {
585+
this.tryMatchPipeOperator();
586+
}
587+
// Try ampersand operator
588+
else if (c === '&') {
589+
const result = this.tryMatchAmpersandOperator();
590+
if (result === false) return false;
591+
}
592+
// Try 'as' keyword operator
593+
else if (c === 'a') {
594+
const result = this.tryMatchAsOperator();
595+
if (!result) return false;
596+
}
597+
// No operator matched
598+
else {
524599
return false;
525600
}
601+
602+
this.pos++;
603+
604+
// All successful branches above set this.current, so it's safe to access
605+
if (this.current && this.isOperatorEnabled(this.current.value as string)) {
606+
return true;
607+
}
608+
609+
this.pos = startPos;
610+
return false;
526611
}
527612

528613
isOperatorEnabled(op: string): boolean {

‎yarn.lock‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,10 @@
4747
dependencies:
4848
"@jridgewell/trace-mapping" "0.3.9"
4949

50-
"@esbuild/win32-x64@0.25.10":
50+
"@esbuild/linux-x64@0.25.10":
5151
version "0.25.10"
52-
resolved "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.10.tgz"
53-
integrity sha512-9KpxSVFCu0iK1owoez6aC/s/EdUQLDN3adTxGCqxMVhrPDj6bt5dbrHDXUuq+Bs2vATFBBrQS5vdQ/Ed2P+nbw==
52+
resolved "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.10.tgz"
53+
integrity sha512-QSX81KhFoZGwenVyPoberggdW1nrQZSvfVDAIUXr3WqLRZGZqWk/P4T8p2SP+de2Sr5HPcvjhcJzEiulKgnxtA==
5454

5555
"@eslint-community/eslint-utils@^4.1.2", "@eslint-community/eslint-utils@^4.4.0", "@eslint-community/eslint-utils@^4.5.0", "@eslint-community/eslint-utils@^4.7.0", "@eslint-community/eslint-utils@^4.8.0":
5656
version "4.9.0"
@@ -303,15 +303,15 @@
303303
estree-walker "^2.0.2"
304304
picomatch "^4.0.2"
305305

306-
"@rollup/rollup-win32-x64-gnu@4.52.0":
306+
"@rollup/rollup-linux-x64-gnu@4.52.0":
307307
version "4.52.0"
308-
resolved "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.52.0.tgz"
309-
integrity sha512-fGk03kQylNaCOQ96HDMeT7E2n91EqvCDd3RwvT5k+xNdFCeMGnj5b5hEgTGrQuyidqSsD3zJDQ21QIaxXqTBJw==
308+
resolved "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.52.0.tgz"
309+
integrity sha512-nmSVN+F2i1yKZ7rJNKO3G7ZzmxJgoQBQZ/6c4MuS553Grmr7WqR7LLDcYG53Z2m9409z3JLt4sCOhLdbKQ3HmA==
310310

311-
"@rollup/rollup-win32-x64-msvc@4.52.0":
311+
"@rollup/rollup-linux-x64-musl@4.52.0":
312312
version "4.52.0"
313-
resolved "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.52.0.tgz"
314-
integrity sha512-6iKDCVSIUQ8jPMoIV0OytRKniaYyy5EbY/RRydmLW8ZR3cEBhxbWl5ro0rkUNe0ef6sScvhbY79HrjRm8i3vDQ==
313+
resolved "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.52.0.tgz"
314+
integrity sha512-2d0qRo33G6TfQVjaMR71P+yJVGODrt5V6+T0BDYH4EMfGgdC/2HWDVjSSFw888GSzAZUwuska3+zxNUCDco6rQ==
315315

316316
"@rtsao/scc@^1.1.0":
317317
version "1.1.0"
@@ -515,10 +515,10 @@
515515
"@typescript-eslint/types" "8.44.0"
516516
eslint-visitor-keys "^4.2.1"
517517

518-
"@typescript/native-preview-win32-x64@7.0.0-dev.20250920.1":
518+
"@typescript/native-preview-linux-x64@7.0.0-dev.20250920.1":
519519
version "7.0.0-dev.20250920.1"
520-
resolved "https://registry.npmjs.org/@typescript/native-preview-win32-x64/-/native-preview-win32-x64-7.0.0-dev.20250920.1.tgz"
521-
integrity sha512-arxNEOP4LMr+rxCxbitG9ejEkmvvj7pkjMFTAFHfWJC0mKuT+1cQu79iZBhY16oHB4HLCziJRBRdMQhIB2Wf+w==
520+
resolved "https://registry.npmjs.org/@typescript/native-preview-linux-x64/-/native-preview-linux-x64-7.0.0-dev.20250920.1.tgz"
521+
integrity sha512-6slYt8MKVAOZDbs2GXznPJ7KVmMLssqvmTJOmmNNyNJt/0MBJqwgL1b/TlxS9UgUTuBsNyKJbLtJgjvBVixIAA==
522522

523523
"@typescript/native-preview@^7.0.0-dev.20250920.1":
524524
version "7.0.0-dev.20250920.1"

0 commit comments

Comments
 (0)