diff --git a/src/hoister.ts b/src/hoister.ts new file mode 100644 index 0000000..ad4099a --- /dev/null +++ b/src/hoister.ts @@ -0,0 +1,171 @@ +import type { Token } from './tokenizer.js' + +export interface HoistNode { + prefix: Token[] + cases: { cond: Token[]; node: HoistNode }[] +} + +/** + * A set of tokens optionally starting with a directive + */ +type PreprocessorSegment = { + directive?: Token[] | undefined + suffix: Token[] + scope: number +} + +const preScopeDelta = { + if: 1, + ifdef: 1, + ifndef: 1, +} as Record + +const postScopeDelta = { + endif: -1, +} as Record + +export function segmentDirectives(tokens: Token[]): PreprocessorSegment[] { + const segments: PreprocessorSegment[] = [] + + // Gathering the first non-directive segment, if it exists + let cursor = 0 + let scope = 0 + const prefix: Token[] = [] + while (cursor < tokens.length && tokens[cursor].value !== '#') { + prefix.push(tokens[cursor++]) + } + + if (prefix.length > 0) { + segments.push({ suffix: trimWhitespace(prefix), scope }) + } + + while (cursor < tokens.length) { + const directive: Token[] = [] + while (cursor < tokens.length && tokens[cursor].value !== '\\') { + directive.push(tokens[cursor++]) + } + directive.push(tokens[cursor++]) // push the '\\' + + const suffix: Token[] = [] + while (cursor < tokens.length && tokens[cursor].value !== '#') { + suffix.push(tokens[cursor++]) + } + + const name = directive[1]?.value ?? '' + scope += preScopeDelta[name] || 0 + segments.push({ directive, suffix: trimWhitespace(suffix), scope }) + scope += postScopeDelta[name] || 0 + } + + return segments +} + +export function getDirectiveName(segment: PreprocessorSegment): string { + return segment.directive?.[1]?.value ?? '' +} + +export function trimWhitespace(tokens: Token[]): Token[] { + let start = 0 + let end = tokens.length - 1 + + if (end === 0 && tokens[0].type === 'whitespace') { + return [] + } + + while (start < end && tokens[start].type === 'whitespace') { + start++ + } + + while (end > start && tokens[end].type === 'whitespace') { + end-- + } + + return tokens.slice(start, end + 1) +} + +function constructHoistTree( + segments: PreprocessorSegment[], + scope: number = 0, + remainder: HoistNode | undefined = undefined, +): HoistNode { + let leafNode: HoistNode = { + cases: remainder?.cases ?? [], + prefix: remainder?.prefix ?? [], + } + + let currentNode: HoistNode | undefined + // The segment index that marks the end of the currently + // explored case (exclusive) + let caseEnd = segments.length - 1 + + for (let seg = segments.length - 1; seg >= 0; seg--) { + const segment = segments[seg] + if (segment.scope === scope) { + // A segment that belongs to the outer scope, meaning we're at the top of the nested node + leafNode.prefix = [...segment.suffix, ...leafNode.prefix] + continue + } + + if (segment.scope > scope + 1) { + continue // A nested segment, skip it + } + + const name = getDirectiveName(segment) + if (name === '') { + // The first segment, no directive + if (leafNode) { + leafNode.prefix = [...segment.suffix, ...leafNode.prefix] + } + } else if (name === 'endif') { + // Prepending the suffix of the endif segment before the leaf node + leafNode.prefix = [...segment.suffix, ...leafNode.prefix] + caseEnd = seg + currentNode = { + cases: [], + prefix: [], + } + } else if (name === 'else' || name === 'elif' || name === 'if' || name === 'ifdef' || name === 'ifndef') { + const caseNode = constructHoistTree(segments.slice(seg, caseEnd), scope + 1, leafNode) + caseEnd = seg // the next ends where the previous started + currentNode?.cases.unshift({ cond: segment.directive ?? [], node: caseNode }) + } + + if (name === 'if' || name === 'ifdef' || name === 'ifndef') { + // We're finishing up a whole node + leafNode = currentNode! + } + } + + // No nested conditions in this node + return leafNode +} + +function flattenHoistNode(node: HoistNode, prefix: Token[]): Token[] { + if (node.cases.length === 0) { + return [...prefix, ...node.prefix] + } + + return [ + ...node.cases.flatMap((case_) => { + return [ + ...case_.cond, + { type: 'whitespace' as const, value: '\n' }, + ...flattenHoistNode(case_.node, [...prefix, ...node.prefix]), + { type: 'whitespace' as const, value: '\n' }, + ] + }), + { type: 'symbol', value: '#' }, + { type: 'keyword', value: 'endif' }, + { type: 'symbol' as const, value: '\\' }, + { type: 'whitespace' as const, value: '\n' }, + ] +} + +/** + * Hoists preprocessor directives on the token level + */ +export function hoistPreprocessorDirectives(tokens: Token[]): Token[] { + const tree = constructHoistTree(segmentDirectives(tokens)) + + return flattenHoistNode(tree, []) +} diff --git a/src/parser.ts b/src/parser.ts index a313fe4..e8f16af 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -38,6 +38,7 @@ import { WhileStatement, LayoutQualifierStatement, } from './ast.js' +import { hoistPreprocessorDirectives } from './hoister.js' import { type Token, tokenize } from './tokenizer.js' // https://engineering.desmos.com/articles/pratt-parser @@ -137,24 +138,87 @@ function getScopeDelta(token: Token): number { return SCOPE_DELTAS[token.value] ?? 0 } -function peek(tokens: Token[], offset: number = 0): Token | null { - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i] - if (token.type !== 'whitespace' && token.type !== 'comment') { - if (offset === 0) return token - else offset-- +interface Tokens { + list: Token[] + /** + * Starts at 0, points at the next token yet to be consumed. + */ + cursor: number + /** + * If a macro (or preprocessor statement) was encountered + * during parsing. Used to workaround expression-level macros + * and transform them into statements. + */ + encounteredMacro?: boolean | undefined +} + +function hasNextToken(tokens: Tokens): boolean { + return tokens.cursor < tokens.list.length +} + +function skipIrrelevant(tokens: Tokens, ignorePreprocessor: boolean = true): void { + let preprocessorScope = 0 + for (; hasNextToken(tokens); tokens.cursor++) { + const token = tokens.list[tokens.cursor] + + if (ignorePreprocessor && token.value === '#') { + tokens.encounteredMacro = true + let name = '' + const nameToken = tokens.list[++tokens.cursor] + if (nameToken.value !== '\\') { + name = nameToken.value + } + + if (name === 'if' || name === 'ifdef' || name === 'ifndef') { + preprocessorScope++ + } else if (name === 'endif') { + preprocessorScope-- + tokens.cursor++ + } else { + // Ignoring everything up until the end of the directive + while (hasNextToken(tokens) && tokens.list[tokens.cursor].value !== '\\') tokens.cursor++ + tokens.cursor++ + } + + continue + } + + if (preprocessorScope > 0) { + continue + } + + if (token.type === 'whitespace' || token.type === 'comment') { + continue + } + + // Something relevant + return + } +} + +function peek(tokens: Tokens, offset: number = 0, ignorePreprocessor: boolean = true): Token | null { + const prevCursor = tokens.cursor + while (hasNextToken(tokens)) { + skipIrrelevant(tokens, ignorePreprocessor) + + if (offset === 0) { + const token = tokens.list[tokens.cursor] + tokens.cursor = prevCursor + return token + } else { + offset-- + tokens.cursor++ } } + tokens.cursor = prevCursor return null } -function consume(tokens: Token[], expected?: string): Token { - // TODO: use token cursor for performance and store for sourcemaps - let token = tokens.shift() - while (token && (token.type === 'whitespace' || token.type === 'comment')) { - token = tokens.shift() - } +function consume(tokens: Tokens, expected?: string, ignorePreprocessor: boolean = true): Token { + // TODO: use store for sourcemaps + skipIrrelevant(tokens, ignorePreprocessor) + const token = tokens.list[tokens.cursor++] if (token === undefined && expected !== undefined) { throw new SyntaxError(`Expected "${expected}"`) @@ -167,7 +231,7 @@ function consume(tokens: Token[], expected?: string): Token { return token } -function parseExpression(tokens: Token[], minBindingPower: number = 0): Expression { +function parseExpression(tokens: Tokens, minBindingPower: number = 0): Expression { let token = consume(tokens) let lhs: Expression @@ -190,7 +254,7 @@ function parseExpression(tokens: Token[], minBindingPower: number = 0): Expressi throw new SyntaxError(`Unexpected token: "${token.value}"`) } - while (tokens.length) { + while (hasNextToken(tokens)) { token = peek(tokens)! if (token.value in POSTFIX_OPERATOR_PRECEDENCE) { @@ -282,7 +346,7 @@ function parseExpression(tokens: Token[], minBindingPower: number = 0): Expressi return lhs } -function parseTypeSpecifier(tokens: Token[]): Identifier | ArraySpecifier { +function parseTypeSpecifier(tokens: Tokens): Identifier | ArraySpecifier { let typeSpecifier: Identifier | ArraySpecifier = { type: 'Identifier', name: consume(tokens).value } if (peek(tokens)?.value === '[') { @@ -311,7 +375,7 @@ function parseTypeSpecifier(tokens: Token[]): Identifier | ArraySpecifier { } function parseVariableDeclarator( - tokens: Token[], + tokens: Tokens, typeSpecifier: Identifier | ArraySpecifier, qualifiers: (ConstantQualifier | InterpolationQualifier | StorageQualifier | PrecisionQualifier)[], layout: Record | null, @@ -329,7 +393,7 @@ function parseVariableDeclarator( } function parseVariable( - tokens: Token[], + tokens: Tokens, typeSpecifier: Identifier | ArraySpecifier, qualifiers: (ConstantQualifier | InterpolationQualifier | StorageQualifier | PrecisionQualifier)[] = [], layout: Record | null = null, @@ -337,7 +401,7 @@ function parseVariable( const declarations: VariableDeclarator[] = [] if (peek(tokens)?.value !== ';') { - while (tokens.length) { + while (hasNextToken(tokens)) { declarations.push(parseVariableDeclarator(tokens, typeSpecifier, qualifiers, layout)) if (peek(tokens)?.value === ',') { @@ -354,7 +418,7 @@ function parseVariable( } function parseBufferInterface( - tokens: Token[], + tokens: Tokens, typeSpecifier: Identifier | ArraySpecifier, qualifiers: LayoutQualifier[] = [], layout: Record | null = null, @@ -369,7 +433,7 @@ function parseBufferInterface( } function parseFunction( - tokens: Token[], + tokens: Tokens, typeSpecifier: ArraySpecifier | Identifier, qualifiers: PrecisionQualifier[] = [], ): FunctionDeclaration { @@ -405,13 +469,13 @@ function parseFunction( return { type: 'FunctionDeclaration', id, qualifiers, typeSpecifier, params, body } } -function parseLayoutQualifier(tokens: Token[], layout: Record): LayoutQualifierStatement { +function parseLayoutQualifier(tokens: Tokens, layout: Record): LayoutQualifierStatement { const qualifier = consume(tokens).value as StorageQualifier consume(tokens, ';') return { type: 'LayoutQualifierStatement', layout, qualifier } } -function parseInvariant(tokens: Token[]): InvariantQualifierStatement { +function parseInvariant(tokens: Tokens): InvariantQualifierStatement { consume(tokens, 'invariant') const typeSpecifier = parseExpression(tokens) as Identifier consume(tokens, ';') @@ -419,7 +483,7 @@ function parseInvariant(tokens: Token[]): InvariantQualifierStatement { } function parseIndeterminate( - tokens: Token[], + tokens: Tokens, ): | VariableDeclaration | FunctionDeclaration @@ -490,12 +554,12 @@ function parseIndeterminate( } } -function parseStruct(tokens: Token[]): StructDeclaration { +function parseStruct(tokens: Tokens): StructDeclaration { consume(tokens, 'struct') const id: Identifier = { type: 'Identifier', name: consume(tokens).value } consume(tokens, '{') const members: VariableDeclaration[] = [] - while (peek(tokens) && peek(tokens)!.value !== '}') { + while (peek(tokens, 0, false) && peek(tokens, 0, false)!.value !== '}') { members.push(...(parseStatements(tokens) as unknown as VariableDeclaration[])) } consume(tokens, '}') @@ -505,7 +569,7 @@ function parseStruct(tokens: Token[]): StructDeclaration { if (peek(tokens)?.type === 'identifier') { const type = id.name const name = consume(tokens).value - tokens.push( + tokens.list.push( { type: 'identifier', value: type }, { type: 'identifier', value: name }, { type: 'symbol', value: ';' }, @@ -517,28 +581,28 @@ function parseStruct(tokens: Token[]): StructDeclaration { return { type: 'StructDeclaration', id, members } } -function parseContinue(tokens: Token[]): ContinueStatement { +function parseContinue(tokens: Tokens): ContinueStatement { consume(tokens, 'continue') consume(tokens, ';') return { type: 'ContinueStatement' } } -function parseBreak(tokens: Token[]): BreakStatement { +function parseBreak(tokens: Tokens): BreakStatement { consume(tokens, 'break') consume(tokens, ';') return { type: 'BreakStatement' } } -function parseDiscard(tokens: Token[]): DiscardStatement { +function parseDiscard(tokens: Tokens): DiscardStatement { consume(tokens, 'discard') consume(tokens, ';') return { type: 'DiscardStatement' } } -function parseReturn(tokens: Token[]): ReturnStatement { +function parseReturn(tokens: Tokens): ReturnStatement { consume(tokens, 'return') let argument: Expression | null = null @@ -548,7 +612,7 @@ function parseReturn(tokens: Token[]): ReturnStatement { return { type: 'ReturnStatement', argument } } -function parseIf(tokens: Token[]): IfStatement { +function parseIf(tokens: Tokens): IfStatement { consume(tokens, 'if') consume(tokens, '(') const test = parseExpression(tokens) @@ -571,7 +635,7 @@ function parseIf(tokens: Token[]): IfStatement { return { type: 'IfStatement', test, consequent, alternate } } -function parseWhile(tokens: Token[]): WhileStatement { +function parseWhile(tokens: Tokens): WhileStatement { consume(tokens, 'while') consume(tokens, '(') const test = parseExpression(tokens) @@ -581,7 +645,7 @@ function parseWhile(tokens: Token[]): WhileStatement { return { type: 'WhileStatement', test, body } } -function parseFor(tokens: Token[]): ForStatement { +function parseFor(tokens: Tokens): ForStatement { consume(tokens, 'for') consume(tokens, '(') const typeSpecifier = parseExpression(tokens) as Identifier | ArraySpecifier @@ -596,7 +660,7 @@ function parseFor(tokens: Token[]): ForStatement { return { type: 'ForStatement', init, test, update, body } } -function parseDoWhile(tokens: Token[]): DoWhileStatement { +function parseDoWhile(tokens: Tokens): DoWhileStatement { consume(tokens, 'do') const body = parseBlockOrStatement(tokens) consume(tokens, 'while') @@ -608,12 +672,12 @@ function parseDoWhile(tokens: Token[]): DoWhileStatement { return { type: 'DoWhileStatement', test, body } } -function parseSwitch(tokens: Token[]): SwitchStatement { +function parseSwitch(tokens: Tokens): SwitchStatement { consume(tokens, 'switch') const discriminant = parseExpression(tokens) const cases: SwitchCase[] = [] - while (tokens.length) { + while (hasNextToken(tokens)) { const token = consume(tokens) if (token.value === '}') break @@ -633,7 +697,7 @@ function parseSwitch(tokens: Token[]): SwitchStatement { return { type: 'SwitchStatement', discriminant, cases } } -function parsePrecision(tokens: Token[]): PrecisionQualifierStatement { +function parsePrecision(tokens: Tokens): PrecisionQualifierStatement { consume(tokens, 'precision') const precision = consume(tokens).value as PrecisionQualifier const typeSpecifier: Identifier = { type: 'Identifier', name: consume(tokens).value } @@ -641,8 +705,8 @@ function parsePrecision(tokens: Token[]): PrecisionQualifierStatement { return { type: 'PrecisionQualifierStatement', precision, typeSpecifier } } -function parsePreprocessor(tokens: Token[]): PreprocessorStatement { - consume(tokens, '#') +function parsePreprocessor(tokens: Tokens): PreprocessorStatement { + consume(tokens, '#', false) let name = '' // name can be unset for the # directive which is ignored let value: Expression[] | null = null @@ -694,7 +758,7 @@ function parsePreprocessor(tokens: Token[]): PreprocessorStatement { return { type: 'PreprocessorStatement', name, value } } -function isVariable(tokens: Token[]): boolean { +function isVariable(tokens: Tokens): boolean { let token = peek(tokens, 0) // Skip first token if EOF or not type qualifier/specifier @@ -721,8 +785,8 @@ function isVariable(tokens: Token[]): boolean { return peek(tokens, i)?.type !== 'symbol' } -function parseStatement(tokens: Token[]): Statement { - const token = peek(tokens)! +function parseStatement(tokens: Tokens): Statement { + const token = peek(tokens, 0, false)! let statement: Statement | null = null if (token.value === '#') statement = parsePreprocessor(tokens) @@ -749,25 +813,34 @@ function parseStatement(tokens: Token[]): Statement { return statement } -function parseStatements(tokens: Token[]): Statement[] { +function parseStatements(tokens: Tokens): Statement[] { const body: Statement[] = [] let scopeIndex = 0 while (true) { - const token = peek(tokens) + const token = peek(tokens, 0, false) if (!token) break scopeIndex += getScopeDelta(token) if (scopeIndex < 0 || token.value === '}') break if (token.value === 'case' || token.value === 'default') break - body.push(parseStatement(tokens)) + + tokens.encounteredMacro = false + const start = tokens.cursor + const statement = parseStatement(tokens) + if (tokens.encounteredMacro) { + const hoistedTokens = hoistPreprocessorDirectives(tokens.list.slice(start, tokens.cursor)) + body.push(...parseStatements({ list: hoistedTokens, cursor: 0 })) + } else { + body.push(statement) + } } return body } -function parseBlock(tokens: Token[]): BlockStatement { +function parseBlock(tokens: Tokens): BlockStatement { consume(tokens, '{') const body = parseStatements(tokens) consume(tokens, '}') @@ -775,7 +848,7 @@ function parseBlock(tokens: Token[]): BlockStatement { } // TODO: validate block versus sub-statements for GLSL/WGSL -function parseBlockOrStatement(tokens: Token[]): BlockStatement | Statement { +function parseBlockOrStatement(tokens: Tokens): BlockStatement | Statement { if (peek(tokens)?.value === '{') { return parseBlock(tokens) } else { @@ -796,7 +869,10 @@ export function parse(code: string): Program { // Escape newlines after directives, skip comments code = code.replace(DIRECTIVE_REGEX, '$1\\$2') - const tokens = tokenize(code) + const tokens = { + list: tokenize(code), + cursor: 0, + } return { type: 'Program', body: parseStatements(tokens) } } diff --git a/tests/__snapshots__/generator.test.ts.snap b/tests/__snapshots__/generator.test.ts.snap index e391999..6addfa9 100644 --- a/tests/__snapshots__/generator.test.ts.snap +++ b/tests/__snapshots__/generator.test.ts.snap @@ -16,7 +16,15 @@ int y; #else float z; #endif -};struct LightData{float intensity;vec3 position;float one,two;};uniform LightData Light[4];invariant pc_FragColor;void main(){vec4 lightNormal=vec4(Light[0].position.xyz*Light[0].intensity,0.0);vec4 clipPosition=projectionMatrix*modelViewMatrix*vec4(0,0,0,1);vec4 clipPositionGlobals=globals.projectionMatrix*globals.modelViewMatrix*vec4(0,0,0,1);if(false){}pc_FragColor=vec4(texture(map,vUv).rgb,0.0);float bar=0.0;pc_FragColor.a+=1.0+bar;}float foo; +};struct LightData{float intensity;vec3 position;float one,two;};uniform LightData Light[4];invariant pc_FragColor;void main(){ +#if defined(USE_NORMALMAP) +mat3 tbn=getTangentFrame(-vViewPosition,normal,vNormalMapUv); +#elif defined(USE_CLEARCOAT_NORMALMAP) +mat3 tbn=getTangentFrame(-vViewPosition,normal,vClearcoatNormalMapUv); +#else +mat3 tbn=getTangentFrame(-vViewPosition,normal,vUv); +#endif +vec4 lightNormal=vec4(Light[0].position.xyz*Light[0].intensity,0.0);vec4 clipPosition=projectionMatrix*modelViewMatrix*vec4(0,0,0,1);vec4 clipPositionGlobals=globals.projectionMatrix*globals.modelViewMatrix*vec4(0,0,0,1);if(false){}pc_FragColor=vec4(texture(map,vUv).rgb,0.0);float bar=0.0;pc_FragColor.a+=1.0+bar;}float foo; # #define test #undef test diff --git a/tests/generator.test.ts b/tests/generator.test.ts index 784b76c..d7f4f24 100644 --- a/tests/generator.test.ts +++ b/tests/generator.test.ts @@ -58,6 +58,15 @@ const glsl = /* glsl */ `#version 300 es invariant pc_FragColor; void main() { + mat3 tbn = getTangentFrame(-vViewPosition, normal, + #if defined(USE_NORMALMAP) + vNormalMapUv + #elif defined(USE_CLEARCOAT_NORMALMAP) + vClearcoatNormalMapUv + #else + vUv + #endif + ); vec4 lightNormal = vec4(Light[0].position.xyz * Light[0].intensity, 0.0); vec4 clipPosition = projectionMatrix * modelViewMatrix * vec4(0, 0, 0, 1); vec4 clipPositionGlobals = globals.projectionMatrix * globals.modelViewMatrix * vec4(0, 0, 0, 1); diff --git a/tests/hoister.test.ts b/tests/hoister.test.ts new file mode 100644 index 0000000..85be0f9 --- /dev/null +++ b/tests/hoister.test.ts @@ -0,0 +1,182 @@ +import { type Token, tokenize } from 'shaderkit' +import { describe, expect, it } from 'vitest' +import { hoistPreprocessorDirectives } from '../src/hoister.js' + +const NEWLINE_REGEX = /\\\s+/gm +const DIRECTIVE_REGEX = /(^\s*#[^\\]*?)(\n|\/[\/\*])/gm + +function workAroundDirectiveEnd(code: string): string { + // Fold newlines + code = code.replace(NEWLINE_REGEX, '') + + // Escape newlines after directives, skip comments + code = code.replace(DIRECTIVE_REGEX, '$1\\$2') + + return code +} + +function print(tokens: Token[]) { + let result = '' + let skipNextBaskslash = false + for (const token of tokens) { + if (token.value === '#') { + skipNextBaskslash = true + } + + if (token.value === '\\' && skipNextBaskslash) { + skipNextBaskslash = false + continue + } + + result += token.value + } + return result +} + +const glslComplexCondition = workAroundDirectiveEnd(`\ +mat3 tbn = getTangentFrame(-vViewPosition, normal, +#if defined(USE_NORMALMAP) + vNormalMapUv +#elif defined(USE_CLEARCOAT_NORMALMAP) + vClearcoatNormalMapUv +#else + vUv +#endif +);`) + +const glslSiblingConditions = workAroundDirectiveEnd(`\ +vec3 color = getColor( + #ifdef VIEW_NORMALMAP + normalMap, + #else + colorMap, + #endif + #ifdef HIGH_P + vUvHigh + #else + vUvLow + #endif +);`) + +const glslNestedConditions = workAroundDirectiveEnd(`\ +vec3 color = getColor( + #ifdef VIEW_NORMALMAP + normalMap + #else + #ifdef GRAYSCALE + grayscaleMap + #else + colorMap + #endif + #endif +);`) + +const glslSiblingNestedConditions = workAroundDirectiveEnd(`\ +vec3 color = + #if CACHE + getColor( + #if COLOR + colorMap, + #else + grayscaleMap, + #endif + #else + computeColor( + #endif + #if HIGH_P + highPrecisionUV + #else + lowPrecisionUV + #endif + );`) + +describe('hoistPreprocessorDirectives', () => { + it('hoists complex condition', () => { + const inTokens = tokenize(glslComplexCondition) + const outCode = print(hoistPreprocessorDirectives(inTokens)) + expect(outCode).toMatchInlineSnapshot(` + "#if defined(USE_NORMALMAP) + mat3 tbn = getTangentFrame(-vViewPosition, normal,vNormalMapUv); + #elif defined(USE_CLEARCOAT_NORMALMAP) + mat3 tbn = getTangentFrame(-vViewPosition, normal,vClearcoatNormalMapUv); + #else + mat3 tbn = getTangentFrame(-vViewPosition, normal,vUv); + #endif + " + `) + }) + + it('hoists sibling directives', () => { + const inTokens = tokenize(glslSiblingConditions) + const outCode = print(hoistPreprocessorDirectives(inTokens)) + expect(outCode).toMatchInlineSnapshot(` + "#ifdef VIEW_NORMALMAP + #ifdef HIGH_P + vec3 color = getColor(normalMap,vUvHigh); + #else + vec3 color = getColor(normalMap,vUvLow); + #endif + + #else + #ifdef HIGH_P + vec3 color = getColor(colorMap,vUvHigh); + #else + vec3 color = getColor(colorMap,vUvLow); + #endif + + #endif + " + `) + }) + + it('hoists nested directives', () => { + const inTokens = tokenize(glslNestedConditions) + const outCode = print(hoistPreprocessorDirectives(inTokens)) + expect(outCode).toMatchInlineSnapshot(` + "#ifdef VIEW_NORMALMAP + vec3 color = getColor(normalMap); + #else + #ifdef GRAYSCALE + vec3 color = getColor(grayscaleMap); + #else + vec3 color = getColor(colorMap); + #endif + + #endif + " + `) + }) + + it('hoists nested & sibling conditions', () => { + const inTokens = tokenize(glslSiblingNestedConditions) + const outCode = print(hoistPreprocessorDirectives(inTokens)) + expect(outCode).toMatchInlineSnapshot(` + "#if CACHE + #if COLOR + #if HIGH_P + vec3 color =getColor(colorMap,highPrecisionUV); + #else + vec3 color =getColor(colorMap,lowPrecisionUV); + #endif + + #else + #if HIGH_P + vec3 color =getColor(grayscaleMap,highPrecisionUV); + #else + vec3 color =getColor(grayscaleMap,lowPrecisionUV); + #endif + + #endif + + #else + #if HIGH_P + vec3 color =computeColor(highPrecisionUV); + #else + vec3 color =computeColor(lowPrecisionUV); + #endif + + #endif + " + `) + }) +}) diff --git a/tests/parser.test.ts b/tests/parser.test.ts index e16c54a..11f5360 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -1188,6 +1188,204 @@ describe('parser', () => { ]) }) + it('hoists preprocessor directives out of expressions', () => { + expect( + parse(`\ +mat3 tbn = getTangentFrame(-vViewPosition, normal, +#if defined(USE_NORMALMAP) + vNormalMapUv +#elif defined(USE_CLEARCOAT_NORMALMAP) + vClearcoatNormalMapUv +#else + vUv +#endif +);`).body, + ).toMatchInlineSnapshot(` + [ + { + "name": "if", + "type": "PreprocessorStatement", + "value": [ + { + "arguments": [ + { + "name": "USE_NORMALMAP", + "type": "Identifier", + }, + ], + "callee": { + "name": "defined", + "type": "Identifier", + }, + "type": "CallExpression", + }, + ], + }, + { + "declarations": [ + { + "id": { + "name": "tbn", + "type": "Identifier", + }, + "init": { + "arguments": [ + { + "argument": { + "name": "vViewPosition", + "type": "Identifier", + }, + "operator": "-", + "prefix": true, + "type": "UnaryExpression", + }, + { + "name": "normal", + "type": "Identifier", + }, + { + "name": "vNormalMapUv", + "type": "Identifier", + }, + ], + "callee": { + "name": "getTangentFrame", + "type": "Identifier", + }, + "type": "CallExpression", + }, + "layout": null, + "qualifiers": [], + "type": "VariableDeclarator", + "typeSpecifier": { + "name": "mat3", + "type": "Identifier", + }, + }, + ], + "type": "VariableDeclaration", + }, + { + "name": "elif", + "type": "PreprocessorStatement", + "value": [ + { + "arguments": [ + { + "name": "USE_CLEARCOAT_NORMALMAP", + "type": "Identifier", + }, + ], + "callee": { + "name": "defined", + "type": "Identifier", + }, + "type": "CallExpression", + }, + ], + }, + { + "declarations": [ + { + "id": { + "name": "tbn", + "type": "Identifier", + }, + "init": { + "arguments": [ + { + "argument": { + "name": "vViewPosition", + "type": "Identifier", + }, + "operator": "-", + "prefix": true, + "type": "UnaryExpression", + }, + { + "name": "normal", + "type": "Identifier", + }, + { + "name": "vClearcoatNormalMapUv", + "type": "Identifier", + }, + ], + "callee": { + "name": "getTangentFrame", + "type": "Identifier", + }, + "type": "CallExpression", + }, + "layout": null, + "qualifiers": [], + "type": "VariableDeclarator", + "typeSpecifier": { + "name": "mat3", + "type": "Identifier", + }, + }, + ], + "type": "VariableDeclaration", + }, + { + "name": "else", + "type": "PreprocessorStatement", + "value": null, + }, + { + "declarations": [ + { + "id": { + "name": "tbn", + "type": "Identifier", + }, + "init": { + "arguments": [ + { + "argument": { + "name": "vViewPosition", + "type": "Identifier", + }, + "operator": "-", + "prefix": true, + "type": "UnaryExpression", + }, + { + "name": "normal", + "type": "Identifier", + }, + { + "name": "vUv", + "type": "Identifier", + }, + ], + "callee": { + "name": "getTangentFrame", + "type": "Identifier", + }, + "type": "CallExpression", + }, + "layout": null, + "qualifiers": [], + "type": "VariableDeclarator", + "typeSpecifier": { + "name": "mat3", + "type": "Identifier", + }, + }, + ], + "type": "VariableDeclaration", + }, + { + "name": "endif", + "type": "PreprocessorStatement", + "value": null, + }, + ] + `) + }) + it('parses nested block statements', () => { expect(parse('{ { float a = float(a) + 1.0; } }\n').body).toStrictEqual<[BlockStatement]>([ {