From f3b0a5d11c94287e17c646551605029abfb98d77 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Fri, 12 Dec 2025 14:26:22 +0100 Subject: [PATCH 1/9] chore: Use cursor instead of shifting tokens --- src/parser.ts | 89 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index a313fe4..a3846fe 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -137,9 +137,21 @@ function getScopeDelta(token: Token): number { return SCOPE_DELTAS[token.value] ?? 0 } -function peek(tokens: Token[], offset: number = 0): Token | null { - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i] +interface Tokens { + list: Token[] + /** + * Starts at 0, points at the next token yet to be consumed. + */ + cursor: number +} + +function hasNextToken(tokens: Tokens): boolean { + return tokens.cursor < tokens.list.length +} + +function peek(tokens: Tokens, offset: number = 0): Token | null { + for (let i = tokens.cursor; i < tokens.list.length; i++) { + const token = tokens.list[i] if (token.type !== 'whitespace' && token.type !== 'comment') { if (offset === 0) return token else offset-- @@ -149,11 +161,11 @@ function peek(tokens: Token[], offset: number = 0): Token | null { return null } -function consume(tokens: Token[], expected?: string): Token { +function consume(tokens: Tokens, expected?: string): Token { // TODO: use token cursor for performance and store for sourcemaps - let token = tokens.shift() + let token = tokens.list[tokens.cursor++] while (token && (token.type === 'whitespace' || token.type === 'comment')) { - token = tokens.shift() + token = tokens.list[tokens.cursor++] } if (token === undefined && expected !== undefined) { @@ -167,7 +179,7 @@ function consume(tokens: Token[], expected?: string): Token { return token } -function parseExpression(tokens: Token[], minBindingPower: number = 0): Expression { +function parseExpression(tokens: Tokens, minBindingPower: number = 0): Expression { let token = consume(tokens) let lhs: Expression @@ -190,7 +202,7 @@ function parseExpression(tokens: Token[], minBindingPower: number = 0): Expressi throw new SyntaxError(`Unexpected token: "${token.value}"`) } - while (tokens.length) { + while (hasNextToken(tokens)) { token = peek(tokens)! if (token.value in POSTFIX_OPERATOR_PRECEDENCE) { @@ -282,7 +294,7 @@ function parseExpression(tokens: Token[], minBindingPower: number = 0): Expressi return lhs } -function parseTypeSpecifier(tokens: Token[]): Identifier | ArraySpecifier { +function parseTypeSpecifier(tokens: Tokens): Identifier | ArraySpecifier { let typeSpecifier: Identifier | ArraySpecifier = { type: 'Identifier', name: consume(tokens).value } if (peek(tokens)?.value === '[') { @@ -311,7 +323,7 @@ function parseTypeSpecifier(tokens: Token[]): Identifier | ArraySpecifier { } function parseVariableDeclarator( - tokens: Token[], + tokens: Tokens, typeSpecifier: Identifier | ArraySpecifier, qualifiers: (ConstantQualifier | InterpolationQualifier | StorageQualifier | PrecisionQualifier)[], layout: Record | null, @@ -329,7 +341,7 @@ function parseVariableDeclarator( } function parseVariable( - tokens: Token[], + tokens: Tokens, typeSpecifier: Identifier | ArraySpecifier, qualifiers: (ConstantQualifier | InterpolationQualifier | StorageQualifier | PrecisionQualifier)[] = [], layout: Record | null = null, @@ -337,7 +349,7 @@ function parseVariable( const declarations: VariableDeclarator[] = [] if (peek(tokens)?.value !== ';') { - while (tokens.length) { + while (hasNextToken(tokens)) { declarations.push(parseVariableDeclarator(tokens, typeSpecifier, qualifiers, layout)) if (peek(tokens)?.value === ',') { @@ -354,7 +366,7 @@ function parseVariable( } function parseBufferInterface( - tokens: Token[], + tokens: Tokens, typeSpecifier: Identifier | ArraySpecifier, qualifiers: LayoutQualifier[] = [], layout: Record | null = null, @@ -369,7 +381,7 @@ function parseBufferInterface( } function parseFunction( - tokens: Token[], + tokens: Tokens, typeSpecifier: ArraySpecifier | Identifier, qualifiers: PrecisionQualifier[] = [], ): FunctionDeclaration { @@ -405,13 +417,13 @@ function parseFunction( return { type: 'FunctionDeclaration', id, qualifiers, typeSpecifier, params, body } } -function parseLayoutQualifier(tokens: Token[], layout: Record): LayoutQualifierStatement { +function parseLayoutQualifier(tokens: Tokens, layout: Record): LayoutQualifierStatement { const qualifier = consume(tokens).value as StorageQualifier consume(tokens, ';') return { type: 'LayoutQualifierStatement', layout, qualifier } } -function parseInvariant(tokens: Token[]): InvariantQualifierStatement { +function parseInvariant(tokens: Tokens): InvariantQualifierStatement { consume(tokens, 'invariant') const typeSpecifier = parseExpression(tokens) as Identifier consume(tokens, ';') @@ -419,7 +431,7 @@ function parseInvariant(tokens: Token[]): InvariantQualifierStatement { } function parseIndeterminate( - tokens: Token[], + tokens: Tokens, ): | VariableDeclaration | FunctionDeclaration @@ -490,7 +502,7 @@ function parseIndeterminate( } } -function parseStruct(tokens: Token[]): StructDeclaration { +function parseStruct(tokens: Tokens): StructDeclaration { consume(tokens, 'struct') const id: Identifier = { type: 'Identifier', name: consume(tokens).value } consume(tokens, '{') @@ -505,7 +517,7 @@ function parseStruct(tokens: Token[]): StructDeclaration { if (peek(tokens)?.type === 'identifier') { const type = id.name const name = consume(tokens).value - tokens.push( + tokens.list.push( { type: 'identifier', value: type }, { type: 'identifier', value: name }, { type: 'symbol', value: ';' }, @@ -517,28 +529,28 @@ function parseStruct(tokens: Token[]): StructDeclaration { return { type: 'StructDeclaration', id, members } } -function parseContinue(tokens: Token[]): ContinueStatement { +function parseContinue(tokens: Tokens): ContinueStatement { consume(tokens, 'continue') consume(tokens, ';') return { type: 'ContinueStatement' } } -function parseBreak(tokens: Token[]): BreakStatement { +function parseBreak(tokens: Tokens): BreakStatement { consume(tokens, 'break') consume(tokens, ';') return { type: 'BreakStatement' } } -function parseDiscard(tokens: Token[]): DiscardStatement { +function parseDiscard(tokens: Tokens): DiscardStatement { consume(tokens, 'discard') consume(tokens, ';') return { type: 'DiscardStatement' } } -function parseReturn(tokens: Token[]): ReturnStatement { +function parseReturn(tokens: Tokens): ReturnStatement { consume(tokens, 'return') let argument: Expression | null = null @@ -548,7 +560,7 @@ function parseReturn(tokens: Token[]): ReturnStatement { return { type: 'ReturnStatement', argument } } -function parseIf(tokens: Token[]): IfStatement { +function parseIf(tokens: Tokens): IfStatement { consume(tokens, 'if') consume(tokens, '(') const test = parseExpression(tokens) @@ -571,7 +583,7 @@ function parseIf(tokens: Token[]): IfStatement { return { type: 'IfStatement', test, consequent, alternate } } -function parseWhile(tokens: Token[]): WhileStatement { +function parseWhile(tokens: Tokens): WhileStatement { consume(tokens, 'while') consume(tokens, '(') const test = parseExpression(tokens) @@ -581,7 +593,7 @@ function parseWhile(tokens: Token[]): WhileStatement { return { type: 'WhileStatement', test, body } } -function parseFor(tokens: Token[]): ForStatement { +function parseFor(tokens: Tokens): ForStatement { consume(tokens, 'for') consume(tokens, '(') const typeSpecifier = parseExpression(tokens) as Identifier | ArraySpecifier @@ -596,7 +608,7 @@ function parseFor(tokens: Token[]): ForStatement { return { type: 'ForStatement', init, test, update, body } } -function parseDoWhile(tokens: Token[]): DoWhileStatement { +function parseDoWhile(tokens: Tokens): DoWhileStatement { consume(tokens, 'do') const body = parseBlockOrStatement(tokens) consume(tokens, 'while') @@ -608,12 +620,12 @@ function parseDoWhile(tokens: Token[]): DoWhileStatement { return { type: 'DoWhileStatement', test, body } } -function parseSwitch(tokens: Token[]): SwitchStatement { +function parseSwitch(tokens: Tokens): SwitchStatement { consume(tokens, 'switch') const discriminant = parseExpression(tokens) const cases: SwitchCase[] = [] - while (tokens.length) { + while (hasNextToken(tokens)) { const token = consume(tokens) if (token.value === '}') break @@ -633,7 +645,7 @@ function parseSwitch(tokens: Token[]): SwitchStatement { return { type: 'SwitchStatement', discriminant, cases } } -function parsePrecision(tokens: Token[]): PrecisionQualifierStatement { +function parsePrecision(tokens: Tokens): PrecisionQualifierStatement { consume(tokens, 'precision') const precision = consume(tokens).value as PrecisionQualifier const typeSpecifier: Identifier = { type: 'Identifier', name: consume(tokens).value } @@ -641,7 +653,7 @@ function parsePrecision(tokens: Token[]): PrecisionQualifierStatement { return { type: 'PrecisionQualifierStatement', precision, typeSpecifier } } -function parsePreprocessor(tokens: Token[]): PreprocessorStatement { +function parsePreprocessor(tokens: Tokens): PreprocessorStatement { consume(tokens, '#') let name = '' // name can be unset for the # directive which is ignored @@ -694,7 +706,7 @@ function parsePreprocessor(tokens: Token[]): PreprocessorStatement { return { type: 'PreprocessorStatement', name, value } } -function isVariable(tokens: Token[]): boolean { +function isVariable(tokens: Tokens): boolean { let token = peek(tokens, 0) // Skip first token if EOF or not type qualifier/specifier @@ -721,7 +733,7 @@ function isVariable(tokens: Token[]): boolean { return peek(tokens, i)?.type !== 'symbol' } -function parseStatement(tokens: Token[]): Statement { +function parseStatement(tokens: Tokens): Statement { const token = peek(tokens)! let statement: Statement | null = null @@ -749,7 +761,7 @@ function parseStatement(tokens: Token[]): Statement { return statement } -function parseStatements(tokens: Token[]): Statement[] { +function parseStatements(tokens: Tokens): Statement[] { const body: Statement[] = [] let scopeIndex = 0 @@ -767,7 +779,7 @@ function parseStatements(tokens: Token[]): Statement[] { return body } -function parseBlock(tokens: Token[]): BlockStatement { +function parseBlock(tokens: Tokens): BlockStatement { consume(tokens, '{') const body = parseStatements(tokens) consume(tokens, '}') @@ -775,7 +787,7 @@ function parseBlock(tokens: Token[]): BlockStatement { } // TODO: validate block versus sub-statements for GLSL/WGSL -function parseBlockOrStatement(tokens: Token[]): BlockStatement | Statement { +function parseBlockOrStatement(tokens: Tokens): BlockStatement | Statement { if (peek(tokens)?.value === '{') { return parseBlock(tokens) } else { @@ -796,7 +808,10 @@ export function parse(code: string): Program { // Escape newlines after directives, skip comments code = code.replace(DIRECTIVE_REGEX, '$1\\$2') - const tokens = tokenize(code) + const tokens = { + list: tokenize(code), + cursor: 0, + } return { type: 'Program', body: parseStatements(tokens) } } From 5d2cb3e3385b91865ab1e8b9455ff3bf98bd0645 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Fri, 12 Dec 2025 18:42:45 +0100 Subject: [PATCH 2/9] Skip preprocessor directives during expression parsing --- src/parser.ts | 79 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index a3846fe..e86c198 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -143,30 +143,79 @@ interface Tokens { * Starts at 0, points at the next token yet to be consumed. */ cursor: number + /** + * If a macro (or preprocessor statement) was encountered + * during parsing. Used to workaround expression-level macros + * and transform them into statements. + */ + encounteredMacro?: boolean | undefined } function hasNextToken(tokens: Tokens): boolean { return tokens.cursor < tokens.list.length } -function peek(tokens: Tokens, offset: number = 0): Token | null { - for (let i = tokens.cursor; i < tokens.list.length; i++) { - const token = tokens.list[i] - if (token.type !== 'whitespace' && token.type !== 'comment') { - if (offset === 0) return token - else offset-- +function skipIrrelevant(tokens: Tokens, ignorePreprocessor: boolean = true): void { + let preprocessorScope = 0 + for (; hasNextToken(tokens); tokens.cursor++) { + const token = tokens.list[tokens.cursor] + + if (ignorePreprocessor && token.value === '#') { + tokens.encounteredMacro = true + let name = '' + const nameToken = tokens.list[tokens.cursor + 1] + if (nameToken.value !== '\\') { + name = nameToken.value + } + + if (name === 'if' || name === 'ifdef' || name === 'ifndef') { + preprocessorScope++ + } else if (name === 'endif') { + preprocessorScope-- + } else { + // Ignoring everything up until the end of the directive + while (hasNextToken(tokens) && tokens.list[tokens.cursor].value !== '\\') tokens.cursor++ + } + + continue + } + + if (preprocessorScope > 0) { + continue + } + + if (token.type === 'whitespace' || token.type === 'comment') { + continue + } + + // Something relevant + return + } +} + +function peek(tokens: Tokens, offset: number = 0, ignorePreprocessor: boolean = true): Token | null { + const prevCursor = tokens.cursor + while (hasNextToken(tokens)) { + skipIrrelevant(tokens, ignorePreprocessor) + + if (offset === 0) { + const token = tokens.list[tokens.cursor] + tokens.cursor = prevCursor + return token + } else { + offset-- + tokens.cursor++ } } + tokens.cursor = prevCursor return null } -function consume(tokens: Tokens, expected?: string): Token { - // TODO: use token cursor for performance and store for sourcemaps - let token = tokens.list[tokens.cursor++] - while (token && (token.type === 'whitespace' || token.type === 'comment')) { - token = tokens.list[tokens.cursor++] - } +function consume(tokens: Tokens, expected?: string, ignorePreprocessor: boolean = true): Token { + // TODO: use store for sourcemaps + skipIrrelevant(tokens, ignorePreprocessor) + const token = tokens.list[tokens.cursor++] if (token === undefined && expected !== undefined) { throw new SyntaxError(`Expected "${expected}"`) @@ -654,7 +703,7 @@ function parsePrecision(tokens: Tokens): PrecisionQualifierStatement { } function parsePreprocessor(tokens: Tokens): PreprocessorStatement { - consume(tokens, '#') + consume(tokens, '#', false) let name = '' // name can be unset for the # directive which is ignored let value: Expression[] | null = null @@ -734,7 +783,7 @@ function isVariable(tokens: Tokens): boolean { } function parseStatement(tokens: Tokens): Statement { - const token = peek(tokens)! + const token = peek(tokens, 0, false)! let statement: Statement | null = null if (token.value === '#') statement = parsePreprocessor(tokens) @@ -766,7 +815,7 @@ function parseStatements(tokens: Tokens): Statement[] { let scopeIndex = 0 while (true) { - const token = peek(tokens) + const token = peek(tokens, 0, false) if (!token) break scopeIndex += getScopeDelta(token) From 633fcf3aa85a028e025729a0feb0aaf1303b5515 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Fri, 12 Dec 2025 22:30:39 +0100 Subject: [PATCH 3/9] Fix preprocessor directives in structs --- src/parser.ts | 6 +++-- tests/parser.test.ts | 58 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index e86c198..7868b87 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -163,7 +163,7 @@ function skipIrrelevant(tokens: Tokens, ignorePreprocessor: boolean = true): voi if (ignorePreprocessor && token.value === '#') { tokens.encounteredMacro = true let name = '' - const nameToken = tokens.list[tokens.cursor + 1] + const nameToken = tokens.list[++tokens.cursor] if (nameToken.value !== '\\') { name = nameToken.value } @@ -172,9 +172,11 @@ function skipIrrelevant(tokens: Tokens, ignorePreprocessor: boolean = true): voi preprocessorScope++ } else if (name === 'endif') { preprocessorScope-- + tokens.cursor++ } else { // Ignoring everything up until the end of the directive while (hasNextToken(tokens) && tokens.list[tokens.cursor].value !== '\\') tokens.cursor++ + tokens.cursor++ } continue @@ -556,7 +558,7 @@ function parseStruct(tokens: Tokens): StructDeclaration { const id: Identifier = { type: 'Identifier', name: consume(tokens).value } consume(tokens, '{') const members: VariableDeclaration[] = [] - while (peek(tokens) && peek(tokens)!.value !== '}') { + while (peek(tokens, 0, false) && peek(tokens, 0, false)!.value !== '}') { members.push(...(parseStatements(tokens) as unknown as VariableDeclaration[])) } consume(tokens, '}') diff --git a/tests/parser.test.ts b/tests/parser.test.ts index e16c54a..a7cecc0 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -1188,6 +1188,64 @@ describe('parser', () => { ]) }) + it('hoists preprocessor directives out of expressions', () => { + expect( + parse(`\ +mat3 tbn = getTangentFrame(-vViewPosition, normal +#if defined(USE_NORMALMAP) + vNormalMapUv, +#elif defined(USE_CLEARCOAT_NORMALMAP) + vClearcoatNormalMapUv, +#else + vUv, +#endif +);`).body, + ).toMatchInlineSnapshot(` + [ + { + "declarations": [ + { + "id": { + "name": "tbn", + "type": "Identifier", + }, + "init": { + "arguments": [ + { + "argument": { + "name": "vViewPosition", + "type": "Identifier", + }, + "operator": "-", + "prefix": true, + "type": "UnaryExpression", + }, + { + "name": "normal", + "type": "Identifier", + }, + ], + "callee": { + "name": "getTangentFrame", + "type": "Identifier", + }, + "type": "CallExpression", + }, + "layout": null, + "qualifiers": [], + "type": "VariableDeclarator", + "typeSpecifier": { + "name": "mat3", + "type": "Identifier", + }, + }, + ], + "type": "VariableDeclaration", + }, + ] + `) + }) + it('parses nested block statements', () => { expect(parse('{ { float a = float(a) + 1.0; } }\n').body).toStrictEqual<[BlockStatement]>([ { From 7fc04af9dcd660bd697ce916ca80c0724de173f1 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Sat, 13 Dec 2025 14:55:00 +0100 Subject: [PATCH 4/9] Primitive hoisting --- src/hoister.ts | 154 +++++++++++++++++++++++++ src/parser.ts | 21 ++-- src/tokenizer.ts | 27 +++++ tests/__snapshots__/index.test.ts.snap | 36 ++++++ tests/hoister.test.ts | 147 +++++++++++++++++++++++ tests/parser.test.ts | 45 +------- 6 files changed, 376 insertions(+), 54 deletions(-) create mode 100644 src/hoister.ts create mode 100644 tests/hoister.test.ts diff --git a/src/hoister.ts b/src/hoister.ts new file mode 100644 index 0000000..31cd812 --- /dev/null +++ b/src/hoister.ts @@ -0,0 +1,154 @@ +import type { Token } from './tokenizer.js' + +export interface HoistNode { + prefix: Token[] + cases: { cond: Token[]; node: HoistNode }[] +} + +/** + * A set of tokens optionally starting with a directive + */ +type PreprocessorSegment = { + directive?: Token[] | undefined + suffix: Token[] + scope: number +} + +const preScopeDelta = { + if: 1, + ifdef: 1, + ifndef: 1, +} as Record + +const postScopeDelta = { + endif: -1, +} as Record + +export function segmentDirectives(tokens: Token[]): PreprocessorSegment[] { + const segments: PreprocessorSegment[] = [] + + // Gathering the first non-directive segment, if it exists + let cursor = 0 + let scope = 0 + const prefix: Token[] = [] + while (cursor < tokens.length && tokens[cursor].value !== '#') { + prefix.push(tokens[cursor++]) + } + + if (prefix.length > 0) { + segments.push({ suffix: prefix, scope }) + } + + while (cursor < tokens.length) { + const directive: Token[] = [] + while (cursor < tokens.length && tokens[cursor].value !== '\\') { + directive.push(tokens[cursor++]) + } + directive.push(tokens[cursor++]) // push the '\\' + + const suffix: Token[] = [] + while (cursor < tokens.length && tokens[cursor].value !== '#') { + suffix.push(tokens[cursor++]) + } + + const name = directive[1]?.value ?? '' + scope += preScopeDelta[name] || 0 + segments.push({ directive, suffix, scope }) + scope += postScopeDelta[name] || 0 + } + + return segments +} + +export function getDirectiveName(segment: PreprocessorSegment): string { + return segment.directive?.[1]?.value ?? '' +} + +function constructHoistTree( + segments: PreprocessorSegment[], + scope: number = 0, + remainder: HoistNode | undefined = undefined, +): HoistNode { + let seg = segments.length - 1 + + let leafNode: HoistNode = { + cases: remainder?.cases ?? [], + // Merging the suffix of the last segment with the start of the remainder + prefix: [...segments[seg].suffix, ...(remainder?.prefix ?? [])], + } + + if (seg === 0) { + // No more segments to explore + return leafNode + } + + let currentNode: HoistNode | undefined + // The segment index that marks the end of the currently + // explored case (exclusive) + let caseEnd = seg + + while (seg >= 0) { + if (segments[seg].scope > scope + 1) { + continue // A nested segment, skip it + } + + const name = getDirectiveName(segments[seg]) + if (name === '') { + // The first segment, no directive + if (currentNode) { + currentNode.prefix = [...segments[seg].suffix, ...currentNode.prefix] + } + } else if (name === 'endif') { + // Prepending the suffix of the endif segment before the leaf node + leafNode.prefix = [...segments[seg].suffix, ...leafNode.prefix] + caseEnd = seg + currentNode = { + cases: [], + prefix: [], + } + } else if (name === 'else' || name === 'elif' || name === 'if' || name === 'ifdef' || name === 'ifndef') { + const caseStart = seg + const caseNode = constructHoistTree(segments.slice(caseStart, caseEnd), scope + 1, leafNode) + caseEnd = seg // the next ends where the previous started + currentNode?.cases.unshift({ cond: segments[caseStart].directive ?? [], node: caseNode }) + } + + if (name === 'if' || name === 'ifdef' || name === 'ifndef') { + // We're finishing up a whole node + leafNode = currentNode! + } + + seg-- + } + + // No nested conditions in this node + return leafNode +} + +function flattenHoistNode(node: HoistNode, prefix: Token[]): Token[] { + if (node.cases.length === 0) { + return [...prefix, ...node.prefix] + } + + return [ + ...node.cases.flatMap((case_) => { + return [ + ...case_.cond, + { type: 'whitespace' as const, value: '\n' }, + ...flattenHoistNode(case_.node, [...prefix, ...node.prefix]), + ] + }), + { type: 'symbol', value: '#' }, + { type: 'keyword', value: 'endif' }, + { type: 'whitespace', value: '\n' }, + ] +} + +/** + * Hoists preprocessor directives on the token level + */ +export function hoistPreprocessorDirectives(tokens: Token[]): Token[] { + const tree = constructHoistTree(segmentDirectives(tokens)) + + return flattenHoistNode(tree, []) +} diff --git a/src/parser.ts b/src/parser.ts index 7868b87..6fc212e 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -38,6 +38,7 @@ import { WhileStatement, LayoutQualifierStatement, } from './ast.js' +import { hoistPreprocessorDirectives } from './hoister.js' import { type Token, tokenize } from './tokenizer.js' // https://engineering.desmos.com/articles/pratt-parser @@ -824,7 +825,16 @@ function parseStatements(tokens: Tokens): Statement[] { if (scopeIndex < 0 || token.value === '}') break if (token.value === 'case' || token.value === 'default') break - body.push(parseStatement(tokens)) + + tokens.encounteredMacro = false + const start = tokens.cursor + const statement = parseStatement(tokens) + if (tokens.encounteredMacro) { + const hoistedTokens = hoistPreprocessorDirectives(tokens.list.slice(start, tokens.cursor)) + body.push(...parseStatements({ list: hoistedTokens, cursor: 0 })) + } else { + body.push(statement) + } } return body @@ -846,19 +856,10 @@ function parseBlockOrStatement(tokens: Tokens): BlockStatement | Statement { } } -const NEWLINE_REGEX = /\\\s+/gm -const DIRECTIVE_REGEX = /(^\s*#[^\\]*?)(\n|\/[\/\*])/gm - /** * Parses a string of GLSL (WGSL WIP) code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). */ export function parse(code: string): Program { - // Fold newlines - code = code.replace(NEWLINE_REGEX, '') - - // Escape newlines after directives, skip comments - code = code.replace(DIRECTIVE_REGEX, '$1\\$2') - const tokens = { list: tokenize(code), cursor: 0, diff --git a/src/tokenizer.ts b/src/tokenizer.ts index c2876bc..78174fd 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -42,10 +42,19 @@ function matchAsPrefix(regex: RegExp, string: string, start: number): string | u return regex.exec(string)?.[0] } +const NEWLINE_REGEX = /\\\s+/gm +const DIRECTIVE_REGEX = /(^\s*#[^\\]*?)(\n|\/[\/\*])/gm + /** * Tokenizes a string of GLSL or WGSL code. */ export function tokenize(code: string, index: number = 0): Token[] { + // Fold newlines + code = code.replace(NEWLINE_REGEX, '') + + // Escape newlines after directives, skip comments + code = code.replace(DIRECTIVE_REGEX, '$1\\$2') + const [KEYWORDS, SYMBOLS] = WGSL_REGEX.test(code) ? [WGSL_KEYWORDS, WGSL_SYMBOLS] : [GLSL_KEYWORDS, GLSL_SYMBOLS] const KEYWORDS_LIST = new Set(KEYWORDS) @@ -103,3 +112,21 @@ export function tokenize(code: string, index: number = 0): Token[] { return tokens } + +export function print(tokens: Token[]) { + let result = '' + let skipNextBaskslash = false + for (const token of tokens) { + if (token.value === '#') { + skipNextBaskslash = true + } + + if (token.value === '\\' && skipNextBaskslash) { + skipNextBaskslash = false + continue + } + + result += token.value + } + return result +} diff --git a/tests/__snapshots__/index.test.ts.snap b/tests/__snapshots__/index.test.ts.snap index 3f6d193..c064c33 100644 --- a/tests/__snapshots__/index.test.ts.snap +++ b/tests/__snapshots__/index.test.ts.snap @@ -219,6 +219,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "identifier", "value": "es", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " @@ -331,6 +335,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "int", "value": "1", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " @@ -357,6 +365,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "whitespace", "value": " ", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "comment", "value": "// inline comment", @@ -401,6 +413,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "symbol", "value": ")", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " @@ -459,6 +475,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "keyword", "value": "endif", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " @@ -619,6 +639,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "symbol", "value": ">", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " @@ -1000,6 +1024,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "symbol", "value": ")", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " @@ -1034,6 +1062,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "keyword", "value": "else", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " @@ -1068,6 +1100,10 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "keyword", "value": "endif", }, + { + "type": "symbol", + "value": "\\", + }, { "type": "whitespace", "value": " diff --git a/tests/hoister.test.ts b/tests/hoister.test.ts new file mode 100644 index 0000000..5c9e953 --- /dev/null +++ b/tests/hoister.test.ts @@ -0,0 +1,147 @@ +import { print, tokenize } from 'shaderkit' +import { describe, expect, it } from 'vitest' +import { type HoistNode, hoistPreprocessorDirectives, segmentDirectives } from '../src/hoister.js' + +const glslSiblingConditions = `\ +vec3 color = getColor( + #ifdef VIEW_NORMALMAP + normalMap, + #else + colorMap, + #endif + #ifdef HIGH_P + vUvHigh + #else + vUvLow + #endif +);` + +// function printRecursive(obj: HoistNode): unknown { +// return { +// prefix: print(obj.prefix), +// cases: obj.cases.map((case_) => ({ +// cond: print(case_.cond), +// node: printRecursive(case_.node), +// })), +// } +// } + +describe('segmentDirectives', () => { + it('segments directives', () => { + const inTokens = tokenize(glslSiblingConditions) + const segments = segmentDirectives(inTokens) + expect( + segments.map((seg) => ({ + suffix: print(seg.suffix), + directive: seg.directive ? print(seg.directive) : undefined, + })), + ).toMatchInlineSnapshot(` + [ + { + "directive": undefined, + "suffix": "vec3 color = getColor( + ", + }, + { + "directive": "#ifdef VIEW_NORMALMAP", + "suffix": " + normalMap, + ", + }, + { + "directive": "#else", + "suffix": " + colorMap, + ", + }, + { + "directive": "#endif", + "suffix": " + ", + }, + { + "directive": "#ifdef HIGH_P", + "suffix": " + vUvHigh + ", + }, + { + "directive": "#else", + "suffix": " + vUvLow + ", + }, + { + "directive": "#endif", + "suffix": " + );", + }, + ] + `) + }) +}) + +describe('hoistPreprocessorDirectives', () => { + it('hoists preprocessor directives', () => { + const inCode = `\ +vec3 color = getColor( + #ifdef VIEW_NORMALMAP + normalMap, + #else + colorMap, + #endif + #ifdef HIGH_P + vUvHigh + #else + vUvLow + #endif +);` + + const inTokens = tokenize(inCode) + const outCode = print(hoistPreprocessorDirectives(inTokens, 0, inTokens.length)) + expect(outCode).toMatchInlineSnapshot(` + "#ifdef VIEW_NORMALMAP + #ifdef HIGH_P + vec3 color = getColor( + + normalMap, + + + vUvHigh + + ); + );#else + vec3 color = getColor( + + normalMap, + + + vUvLow + + ); + );#endif + #else + #ifdef HIGH_P + vec3 color = getColor( + + colorMap, + + + vUvHigh + + ); + );#else + vec3 color = getColor( + + colorMap, + + + vUvLow + + ); + );#endif + #endif + " + `) + }) +}) diff --git a/tests/parser.test.ts b/tests/parser.test.ts index a7cecc0..e132be4 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -1200,50 +1200,7 @@ mat3 tbn = getTangentFrame(-vViewPosition, normal vUv, #endif );`).body, - ).toMatchInlineSnapshot(` - [ - { - "declarations": [ - { - "id": { - "name": "tbn", - "type": "Identifier", - }, - "init": { - "arguments": [ - { - "argument": { - "name": "vViewPosition", - "type": "Identifier", - }, - "operator": "-", - "prefix": true, - "type": "UnaryExpression", - }, - { - "name": "normal", - "type": "Identifier", - }, - ], - "callee": { - "name": "getTangentFrame", - "type": "Identifier", - }, - "type": "CallExpression", - }, - "layout": null, - "qualifiers": [], - "type": "VariableDeclarator", - "typeSpecifier": { - "name": "mat3", - "type": "Identifier", - }, - }, - ], - "type": "VariableDeclaration", - }, - ] - `) + ).toMatchInlineSnapshot(`[]`) }) it('parses nested block statements', () => { From be0cb0e7d314ec386753406c2885cfcb46746fdb Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Tue, 16 Dec 2025 10:55:01 +0100 Subject: [PATCH 5/9] Emit valid #endif directive tokens --- src/hoister.ts | 59 +++++++---- tests/hoister.test.ts | 230 +++++++++++++++++++++--------------------- tests/parser.test.ts | 193 ++++++++++++++++++++++++++++++++++- 3 files changed, 346 insertions(+), 136 deletions(-) diff --git a/src/hoister.ts b/src/hoister.ts index 31cd812..448cc22 100644 --- a/src/hoister.ts +++ b/src/hoister.ts @@ -36,7 +36,7 @@ export function segmentDirectives(tokens: Token[]): PreprocessorSegment[] { } if (prefix.length > 0) { - segments.push({ suffix: prefix, scope }) + segments.push({ suffix: trimWhitespace(prefix), scope }) } while (cursor < tokens.length) { @@ -53,7 +53,7 @@ export function segmentDirectives(tokens: Token[]): PreprocessorSegment[] { const name = directive[1]?.value ?? '' scope += preScopeDelta[name] || 0 - segments.push({ directive, suffix, scope }) + segments.push({ directive, suffix: trimWhitespace(suffix), scope }) scope += postScopeDelta[name] || 0 } @@ -64,6 +64,25 @@ export function getDirectiveName(segment: PreprocessorSegment): string { return segment.directive?.[1]?.value ?? '' } +export function trimWhitespace(tokens: Token[]): Token[] { + let start = 0 + let end = tokens.length - 1 + + if (end === 0 && tokens[0].type === 'whitespace') { + return [] + } + + while (start < end && tokens[start].type === 'whitespace') { + start++ + } + + while (end > start && tokens[end].type === 'whitespace') { + end-- + } + + return tokens.slice(start, end + 1) +} + function constructHoistTree( segments: PreprocessorSegment[], scope: number = 0, @@ -73,13 +92,7 @@ function constructHoistTree( let leafNode: HoistNode = { cases: remainder?.cases ?? [], - // Merging the suffix of the last segment with the start of the remainder - prefix: [...segments[seg].suffix, ...(remainder?.prefix ?? [])], - } - - if (seg === 0) { - // No more segments to explore - return leafNode + prefix: remainder?.prefix ?? [], } let currentNode: HoistNode | undefined @@ -88,29 +101,37 @@ function constructHoistTree( let caseEnd = seg while (seg >= 0) { - if (segments[seg].scope > scope + 1) { + const segment = segments[seg] + if (segment.scope === scope) { + // A segment that belongs to the outer scope, meaning we're at the top of the nested node + leafNode.prefix = [...segment.suffix, ...leafNode.prefix] + seg-- + continue + } + + if (segment.scope > scope + 1) { + seg-- continue // A nested segment, skip it } - const name = getDirectiveName(segments[seg]) + const name = getDirectiveName(segment) if (name === '') { // The first segment, no directive - if (currentNode) { - currentNode.prefix = [...segments[seg].suffix, ...currentNode.prefix] + if (leafNode) { + leafNode.prefix = [...segment.suffix, ...leafNode.prefix] } } else if (name === 'endif') { // Prepending the suffix of the endif segment before the leaf node - leafNode.prefix = [...segments[seg].suffix, ...leafNode.prefix] + leafNode.prefix = [...segment.suffix, ...leafNode.prefix] caseEnd = seg currentNode = { cases: [], prefix: [], } } else if (name === 'else' || name === 'elif' || name === 'if' || name === 'ifdef' || name === 'ifndef') { - const caseStart = seg - const caseNode = constructHoistTree(segments.slice(caseStart, caseEnd), scope + 1, leafNode) + const caseNode = constructHoistTree(segments.slice(seg, caseEnd), scope + 1, leafNode) caseEnd = seg // the next ends where the previous started - currentNode?.cases.unshift({ cond: segments[caseStart].directive ?? [], node: caseNode }) + currentNode?.cases.unshift({ cond: segment.directive ?? [], node: caseNode }) } if (name === 'if' || name === 'ifdef' || name === 'ifndef') { @@ -136,11 +157,13 @@ function flattenHoistNode(node: HoistNode, prefix: Token[]): Token[] { ...case_.cond, { type: 'whitespace' as const, value: '\n' }, ...flattenHoistNode(case_.node, [...prefix, ...node.prefix]), + { type: 'whitespace' as const, value: '\n' }, ] }), { type: 'symbol', value: '#' }, { type: 'keyword', value: 'endif' }, - { type: 'whitespace', value: '\n' }, + { type: 'symbol' as const, value: '\\' }, + { type: 'whitespace' as const, value: '\n' }, ] } diff --git a/tests/hoister.test.ts b/tests/hoister.test.ts index 5c9e953..77f7775 100644 --- a/tests/hoister.test.ts +++ b/tests/hoister.test.ts @@ -1,6 +1,17 @@ import { print, tokenize } from 'shaderkit' import { describe, expect, it } from 'vitest' -import { type HoistNode, hoistPreprocessorDirectives, segmentDirectives } from '../src/hoister.js' +import { hoistPreprocessorDirectives } from '../src/hoister.js' + +const glslComplexCondition = `\ +mat3 tbn = getTangentFrame(-vViewPosition, normal, +#if defined(USE_NORMALMAP) + vNormalMapUv +#elif defined(USE_CLEARCOAT_NORMALMAP) + vClearcoatNormalMapUv +#else + vUv +#endif +);` const glslSiblingConditions = `\ vec3 color = getColor( @@ -16,130 +27,123 @@ vec3 color = getColor( #endif );` -// function printRecursive(obj: HoistNode): unknown { -// return { -// prefix: print(obj.prefix), -// cases: obj.cases.map((case_) => ({ -// cond: print(case_.cond), -// node: printRecursive(case_.node), -// })), -// } -// } - -describe('segmentDirectives', () => { - it('segments directives', () => { - const inTokens = tokenize(glslSiblingConditions) - const segments = segmentDirectives(inTokens) - expect( - segments.map((seg) => ({ - suffix: print(seg.suffix), - directive: seg.directive ? print(seg.directive) : undefined, - })), - ).toMatchInlineSnapshot(` - [ - { - "directive": undefined, - "suffix": "vec3 color = getColor( - ", - }, - { - "directive": "#ifdef VIEW_NORMALMAP", - "suffix": " - normalMap, - ", - }, - { - "directive": "#else", - "suffix": " - colorMap, - ", - }, - { - "directive": "#endif", - "suffix": " - ", - }, - { - "directive": "#ifdef HIGH_P", - "suffix": " - vUvHigh - ", - }, - { - "directive": "#else", - "suffix": " - vUvLow - ", - }, - { - "directive": "#endif", - "suffix": " - );", - }, - ] - `) - }) -}) - -describe('hoistPreprocessorDirectives', () => { - it('hoists preprocessor directives', () => { - const inCode = `\ +const glslNestedConditions = `\ vec3 color = getColor( #ifdef VIEW_NORMALMAP - normalMap, + normalMap #else - colorMap, + #ifdef GRAYSCALE + grayscaleMap + #else + colorMap + #endif #endif - #ifdef HIGH_P - vUvHigh +);` + +const glslSiblingNestedConditions = `\ +vec3 color = + #if CACHE + getColor( + #if COLOR + colorMap, + #else + grayscaleMap, + #endif #else - vUvLow + computeColor( #endif -);` + #if HIGH_P + highPrecisionUV + #else + lowPrecisionUV + #endif + );` + +describe('hoistPreprocessorDirectives', () => { + it('hoists complex condition', () => { + const inTokens = tokenize(glslComplexCondition) + const outCode = print(hoistPreprocessorDirectives(inTokens)) + expect(outCode).toMatchInlineSnapshot(` + "#if defined(USE_NORMALMAP) + mat3 tbn = getTangentFrame(-vViewPosition, normal,vNormalMapUv); + #elif defined(USE_CLEARCOAT_NORMALMAP) + mat3 tbn = getTangentFrame(-vViewPosition, normal,vClearcoatNormalMapUv); + #else + mat3 tbn = getTangentFrame(-vViewPosition, normal,vUv); + #endif + " + `) + }) - const inTokens = tokenize(inCode) - const outCode = print(hoistPreprocessorDirectives(inTokens, 0, inTokens.length)) + it('hoists sibling directives', () => { + const inTokens = tokenize(glslSiblingConditions) + const outCode = print(hoistPreprocessorDirectives(inTokens)) expect(outCode).toMatchInlineSnapshot(` "#ifdef VIEW_NORMALMAP #ifdef HIGH_P - vec3 color = getColor( - - normalMap, - - - vUvHigh - - ); - );#else - vec3 color = getColor( - - normalMap, - - - vUvLow - - ); - );#endif + vec3 color = getColor(normalMap,vUvHigh); + #else + vec3 color = getColor(normalMap,vUvLow); + #endif + #else #ifdef HIGH_P - vec3 color = getColor( - - colorMap, - - - vUvHigh - - ); - );#else - vec3 color = getColor( - - colorMap, - - - vUvLow - - ); - );#endif + vec3 color = getColor(colorMap,vUvHigh); + #else + vec3 color = getColor(colorMap,vUvLow); + #endif + + #endif + " + `) + }) + + it('hoists nested directives', () => { + const inTokens = tokenize(glslNestedConditions) + const outCode = print(hoistPreprocessorDirectives(inTokens)) + expect(outCode).toMatchInlineSnapshot(` + "#ifdef VIEW_NORMALMAP + vec3 color = getColor(normalMap); + #else + #ifdef GRAYSCALE + vec3 color = getColor(grayscaleMap); + #else + vec3 color = getColor(colorMap); + #endif + + #endif + " + `) + }) + + it('hoists nested & sibling conditions', () => { + const inTokens = tokenize(glslSiblingNestedConditions) + const outCode = print(hoistPreprocessorDirectives(inTokens)) + expect(outCode).toMatchInlineSnapshot(` + "#if CACHE + #if COLOR + #if HIGH_P + vec3 color =getColor(colorMap,highPrecisionUV); + #else + vec3 color =getColor(colorMap,lowPrecisionUV); + #endif + + #else + #if HIGH_P + vec3 color =getColor(grayscaleMap,highPrecisionUV); + #else + vec3 color =getColor(grayscaleMap,lowPrecisionUV); + #endif + + #endif + + #else + #if HIGH_P + vec3 color =computeColor(highPrecisionUV); + #else + vec3 color =computeColor(lowPrecisionUV); + #endif + #endif " `) diff --git a/tests/parser.test.ts b/tests/parser.test.ts index e132be4..11f5360 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -1191,16 +1191,199 @@ describe('parser', () => { it('hoists preprocessor directives out of expressions', () => { expect( parse(`\ -mat3 tbn = getTangentFrame(-vViewPosition, normal +mat3 tbn = getTangentFrame(-vViewPosition, normal, #if defined(USE_NORMALMAP) - vNormalMapUv, + vNormalMapUv #elif defined(USE_CLEARCOAT_NORMALMAP) - vClearcoatNormalMapUv, + vClearcoatNormalMapUv #else - vUv, + vUv #endif );`).body, - ).toMatchInlineSnapshot(`[]`) + ).toMatchInlineSnapshot(` + [ + { + "name": "if", + "type": "PreprocessorStatement", + "value": [ + { + "arguments": [ + { + "name": "USE_NORMALMAP", + "type": "Identifier", + }, + ], + "callee": { + "name": "defined", + "type": "Identifier", + }, + "type": "CallExpression", + }, + ], + }, + { + "declarations": [ + { + "id": { + "name": "tbn", + "type": "Identifier", + }, + "init": { + "arguments": [ + { + "argument": { + "name": "vViewPosition", + "type": "Identifier", + }, + "operator": "-", + "prefix": true, + "type": "UnaryExpression", + }, + { + "name": "normal", + "type": "Identifier", + }, + { + "name": "vNormalMapUv", + "type": "Identifier", + }, + ], + "callee": { + "name": "getTangentFrame", + "type": "Identifier", + }, + "type": "CallExpression", + }, + "layout": null, + "qualifiers": [], + "type": "VariableDeclarator", + "typeSpecifier": { + "name": "mat3", + "type": "Identifier", + }, + }, + ], + "type": "VariableDeclaration", + }, + { + "name": "elif", + "type": "PreprocessorStatement", + "value": [ + { + "arguments": [ + { + "name": "USE_CLEARCOAT_NORMALMAP", + "type": "Identifier", + }, + ], + "callee": { + "name": "defined", + "type": "Identifier", + }, + "type": "CallExpression", + }, + ], + }, + { + "declarations": [ + { + "id": { + "name": "tbn", + "type": "Identifier", + }, + "init": { + "arguments": [ + { + "argument": { + "name": "vViewPosition", + "type": "Identifier", + }, + "operator": "-", + "prefix": true, + "type": "UnaryExpression", + }, + { + "name": "normal", + "type": "Identifier", + }, + { + "name": "vClearcoatNormalMapUv", + "type": "Identifier", + }, + ], + "callee": { + "name": "getTangentFrame", + "type": "Identifier", + }, + "type": "CallExpression", + }, + "layout": null, + "qualifiers": [], + "type": "VariableDeclarator", + "typeSpecifier": { + "name": "mat3", + "type": "Identifier", + }, + }, + ], + "type": "VariableDeclaration", + }, + { + "name": "else", + "type": "PreprocessorStatement", + "value": null, + }, + { + "declarations": [ + { + "id": { + "name": "tbn", + "type": "Identifier", + }, + "init": { + "arguments": [ + { + "argument": { + "name": "vViewPosition", + "type": "Identifier", + }, + "operator": "-", + "prefix": true, + "type": "UnaryExpression", + }, + { + "name": "normal", + "type": "Identifier", + }, + { + "name": "vUv", + "type": "Identifier", + }, + ], + "callee": { + "name": "getTangentFrame", + "type": "Identifier", + }, + "type": "CallExpression", + }, + "layout": null, + "qualifiers": [], + "type": "VariableDeclarator", + "typeSpecifier": { + "name": "mat3", + "type": "Identifier", + }, + }, + ], + "type": "VariableDeclaration", + }, + { + "name": "endif", + "type": "PreprocessorStatement", + "value": null, + }, + ] + `) }) it('parses nested block statements', () => { From 6f7fa158824c5d4af3440d19d12033aa73f16920 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Tue, 16 Dec 2025 11:50:45 +0100 Subject: [PATCH 6/9] Update hoister.ts --- src/hoister.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/hoister.ts b/src/hoister.ts index 448cc22..ad4099a 100644 --- a/src/hoister.ts +++ b/src/hoister.ts @@ -88,8 +88,6 @@ function constructHoistTree( scope: number = 0, remainder: HoistNode | undefined = undefined, ): HoistNode { - let seg = segments.length - 1 - let leafNode: HoistNode = { cases: remainder?.cases ?? [], prefix: remainder?.prefix ?? [], @@ -98,19 +96,17 @@ function constructHoistTree( let currentNode: HoistNode | undefined // The segment index that marks the end of the currently // explored case (exclusive) - let caseEnd = seg + let caseEnd = segments.length - 1 - while (seg >= 0) { + for (let seg = segments.length - 1; seg >= 0; seg--) { const segment = segments[seg] if (segment.scope === scope) { // A segment that belongs to the outer scope, meaning we're at the top of the nested node leafNode.prefix = [...segment.suffix, ...leafNode.prefix] - seg-- continue } if (segment.scope > scope + 1) { - seg-- continue // A nested segment, skip it } @@ -138,8 +134,6 @@ function constructHoistTree( // We're finishing up a whole node leafNode = currentNode! } - - seg-- } // No nested conditions in this node From 9ba18bceb1e453c0f1c0acdac25fc3896c7995a4 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Tue, 16 Dec 2025 13:55:56 +0100 Subject: [PATCH 7/9] Move back directive end workaround to the parser --- src/parser.ts | 9 +++++++ src/tokenizer.ts | 9 ------- tests/__snapshots__/index.test.ts.snap | 36 -------------------------- tests/hoister.test.ts | 29 +++++++++++++++------ 4 files changed, 30 insertions(+), 53 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 6fc212e..e8f16af 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -856,10 +856,19 @@ function parseBlockOrStatement(tokens: Tokens): BlockStatement | Statement { } } +const NEWLINE_REGEX = /\\\s+/gm +const DIRECTIVE_REGEX = /(^\s*#[^\\]*?)(\n|\/[\/\*])/gm + /** * Parses a string of GLSL (WGSL WIP) code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). */ export function parse(code: string): Program { + // Fold newlines + code = code.replace(NEWLINE_REGEX, '') + + // Escape newlines after directives, skip comments + code = code.replace(DIRECTIVE_REGEX, '$1\\$2') + const tokens = { list: tokenize(code), cursor: 0, diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 78174fd..a532f9d 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -42,19 +42,10 @@ function matchAsPrefix(regex: RegExp, string: string, start: number): string | u return regex.exec(string)?.[0] } -const NEWLINE_REGEX = /\\\s+/gm -const DIRECTIVE_REGEX = /(^\s*#[^\\]*?)(\n|\/[\/\*])/gm - /** * Tokenizes a string of GLSL or WGSL code. */ export function tokenize(code: string, index: number = 0): Token[] { - // Fold newlines - code = code.replace(NEWLINE_REGEX, '') - - // Escape newlines after directives, skip comments - code = code.replace(DIRECTIVE_REGEX, '$1\\$2') - const [KEYWORDS, SYMBOLS] = WGSL_REGEX.test(code) ? [WGSL_KEYWORDS, WGSL_SYMBOLS] : [GLSL_KEYWORDS, GLSL_SYMBOLS] const KEYWORDS_LIST = new Set(KEYWORDS) diff --git a/tests/__snapshots__/index.test.ts.snap b/tests/__snapshots__/index.test.ts.snap index c064c33..3f6d193 100644 --- a/tests/__snapshots__/index.test.ts.snap +++ b/tests/__snapshots__/index.test.ts.snap @@ -219,10 +219,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "identifier", "value": "es", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " @@ -335,10 +331,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "int", "value": "1", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " @@ -365,10 +357,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "whitespace", "value": " ", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "comment", "value": "// inline comment", @@ -413,10 +401,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "symbol", "value": ")", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " @@ -475,10 +459,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "keyword", "value": "endif", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " @@ -639,10 +619,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "symbol", "value": ">", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " @@ -1024,10 +1000,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "symbol", "value": ")", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " @@ -1062,10 +1034,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "keyword", "value": "else", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " @@ -1100,10 +1068,6 @@ exports[`tokenize > can tokenize GLSL 1`] = ` "type": "keyword", "value": "endif", }, - { - "type": "symbol", - "value": "\\", - }, { "type": "whitespace", "value": " diff --git a/tests/hoister.test.ts b/tests/hoister.test.ts index 77f7775..0b82324 100644 --- a/tests/hoister.test.ts +++ b/tests/hoister.test.ts @@ -2,7 +2,20 @@ import { print, tokenize } from 'shaderkit' import { describe, expect, it } from 'vitest' import { hoistPreprocessorDirectives } from '../src/hoister.js' -const glslComplexCondition = `\ +const NEWLINE_REGEX = /\\\s+/gm +const DIRECTIVE_REGEX = /(^\s*#[^\\]*?)(\n|\/[\/\*])/gm + +function workAroundDirectiveEnd(code: string): string { + // Fold newlines + code = code.replace(NEWLINE_REGEX, '') + + // Escape newlines after directives, skip comments + code = code.replace(DIRECTIVE_REGEX, '$1\\$2') + + return code +} + +const glslComplexCondition = workAroundDirectiveEnd(`\ mat3 tbn = getTangentFrame(-vViewPosition, normal, #if defined(USE_NORMALMAP) vNormalMapUv @@ -11,9 +24,9 @@ mat3 tbn = getTangentFrame(-vViewPosition, normal, #else vUv #endif -);` +);`) -const glslSiblingConditions = `\ +const glslSiblingConditions = workAroundDirectiveEnd(`\ vec3 color = getColor( #ifdef VIEW_NORMALMAP normalMap, @@ -25,9 +38,9 @@ vec3 color = getColor( #else vUvLow #endif -);` +);`) -const glslNestedConditions = `\ +const glslNestedConditions = workAroundDirectiveEnd(`\ vec3 color = getColor( #ifdef VIEW_NORMALMAP normalMap @@ -38,9 +51,9 @@ vec3 color = getColor( colorMap #endif #endif -);` +);`) -const glslSiblingNestedConditions = `\ +const glslSiblingNestedConditions = workAroundDirectiveEnd(`\ vec3 color = #if CACHE getColor( @@ -57,7 +70,7 @@ vec3 color = #else lowPrecisionUV #endif - );` + );`) describe('hoistPreprocessorDirectives', () => { it('hoists complex condition', () => { From a1c4dbc81dfd4bb85bc3525c7112c22650fa17bd Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Tue, 16 Dec 2025 13:58:13 +0100 Subject: [PATCH 8/9] Printing is only useful in tests --- src/tokenizer.ts | 18 ------------------ tests/hoister.test.ts | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index a532f9d..c2876bc 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -103,21 +103,3 @@ export function tokenize(code: string, index: number = 0): Token[] { return tokens } - -export function print(tokens: Token[]) { - let result = '' - let skipNextBaskslash = false - for (const token of tokens) { - if (token.value === '#') { - skipNextBaskslash = true - } - - if (token.value === '\\' && skipNextBaskslash) { - skipNextBaskslash = false - continue - } - - result += token.value - } - return result -} diff --git a/tests/hoister.test.ts b/tests/hoister.test.ts index 0b82324..85be0f9 100644 --- a/tests/hoister.test.ts +++ b/tests/hoister.test.ts @@ -1,4 +1,4 @@ -import { print, tokenize } from 'shaderkit' +import { type Token, tokenize } from 'shaderkit' import { describe, expect, it } from 'vitest' import { hoistPreprocessorDirectives } from '../src/hoister.js' @@ -15,6 +15,24 @@ function workAroundDirectiveEnd(code: string): string { return code } +function print(tokens: Token[]) { + let result = '' + let skipNextBaskslash = false + for (const token of tokens) { + if (token.value === '#') { + skipNextBaskslash = true + } + + if (token.value === '\\' && skipNextBaskslash) { + skipNextBaskslash = false + continue + } + + result += token.value + } + return result +} + const glslComplexCondition = workAroundDirectiveEnd(`\ mat3 tbn = getTangentFrame(-vViewPosition, normal, #if defined(USE_NORMALMAP) From 25bf45e399a9e5b1cbfcfe53d3e8961c5a7e6b3a Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Tue, 20 Jan 2026 15:06:48 +0100 Subject: [PATCH 9/9] Add normal tangent test case to generator --- tests/__snapshots__/generator.test.ts.snap | 10 +++++++++- tests/generator.test.ts | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/__snapshots__/generator.test.ts.snap b/tests/__snapshots__/generator.test.ts.snap index e391999..6addfa9 100644 --- a/tests/__snapshots__/generator.test.ts.snap +++ b/tests/__snapshots__/generator.test.ts.snap @@ -16,7 +16,15 @@ int y; #else float z; #endif -};struct LightData{float intensity;vec3 position;float one,two;};uniform LightData Light[4];invariant pc_FragColor;void main(){vec4 lightNormal=vec4(Light[0].position.xyz*Light[0].intensity,0.0);vec4 clipPosition=projectionMatrix*modelViewMatrix*vec4(0,0,0,1);vec4 clipPositionGlobals=globals.projectionMatrix*globals.modelViewMatrix*vec4(0,0,0,1);if(false){}pc_FragColor=vec4(texture(map,vUv).rgb,0.0);float bar=0.0;pc_FragColor.a+=1.0+bar;}float foo; +};struct LightData{float intensity;vec3 position;float one,two;};uniform LightData Light[4];invariant pc_FragColor;void main(){ +#if defined(USE_NORMALMAP) +mat3 tbn=getTangentFrame(-vViewPosition,normal,vNormalMapUv); +#elif defined(USE_CLEARCOAT_NORMALMAP) +mat3 tbn=getTangentFrame(-vViewPosition,normal,vClearcoatNormalMapUv); +#else +mat3 tbn=getTangentFrame(-vViewPosition,normal,vUv); +#endif +vec4 lightNormal=vec4(Light[0].position.xyz*Light[0].intensity,0.0);vec4 clipPosition=projectionMatrix*modelViewMatrix*vec4(0,0,0,1);vec4 clipPositionGlobals=globals.projectionMatrix*globals.modelViewMatrix*vec4(0,0,0,1);if(false){}pc_FragColor=vec4(texture(map,vUv).rgb,0.0);float bar=0.0;pc_FragColor.a+=1.0+bar;}float foo; # #define test #undef test diff --git a/tests/generator.test.ts b/tests/generator.test.ts index 784b76c..d7f4f24 100644 --- a/tests/generator.test.ts +++ b/tests/generator.test.ts @@ -58,6 +58,15 @@ const glsl = /* glsl */ `#version 300 es invariant pc_FragColor; void main() { + mat3 tbn = getTangentFrame(-vViewPosition, normal, + #if defined(USE_NORMALMAP) + vNormalMapUv + #elif defined(USE_CLEARCOAT_NORMALMAP) + vClearcoatNormalMapUv + #else + vUv + #endif + ); vec4 lightNormal = vec4(Light[0].position.xyz * Light[0].intensity, 0.0); vec4 clipPosition = projectionMatrix * modelViewMatrix * vec4(0, 0, 0, 1); vec4 clipPositionGlobals = globals.projectionMatrix * globals.modelViewMatrix * vec4(0, 0, 0, 1);