From 76905617c3a10839dc300876cc73d2b9ae36f6b3 Mon Sep 17 00:00:00 2001 From: Rangi42 Date: Fri, 8 May 2026 18:57:52 +0200 Subject: [PATCH 1/3] Optimize `skipToLeadingKeyword` for the common `ViewedContent` case --- src/asm/lexer.cpp | 87 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/src/asm/lexer.cpp b/src/asm/lexer.cpp index 2296ac054..f7be6a402 100644 --- a/src/asm/lexer.cpp +++ b/src/asm/lexer.cpp @@ -2082,47 +2082,92 @@ static Token yylex_RAW() { return Token(T_(YYEOF)); } -// This function is called when capturing `REPT`/`FOR` loops and `MACRO` bodies, -// and when skipping unexecuted `IF`/`ELIF`/`ELSE` blocks and `REPT`/`FOR` loops. -// It expects that these constructs' `ENDC`/`ENDR`/`ENDM` closing tokens are only -// valid at the start of their lines, which enables ignoring everything except -// the leading keyword in lines that have one (as well as line continuations). -// -// The only keywords it needs to recognize are case-insensitive `IF`, `ELIF`, -// `ELSE`, `ENDC`, `REPT`, `FOR`, `ENDR`, and `ENDM` (not `MACRO`). -// -// Note that when these constructs are *evaluated*, they can perform expansions -// (for macro args, interpolations, and macro invocations) which may produce -// tokens that would change how these constructs were captured or skipped, if -// they had been produced during the capture/skip non-evaluating phase. -static Token skipToLeadingKeyword() { - assume(!lexerState->enableExpansions); - +template +static Token skipToLeadingKeyword(PeekFnT peekFn, ShiftFnT shiftFn, NextLineFnT nextLineFn) { for (;;) { + int c = peekFn(); if (lexerState->atLineStart) { lexerState->atLineStart = false; - if (int c = skipChars(isBlankSpace); c == EOF) { + while (isBlankSpace(c)) { + shiftFn(); + c = peekFn(); + } + if (c == EOF) { return Token(T_(YYEOF)); } else if (isLetter(c)) { std::string keyword(1, c); - for (c = nextChar(); continuesIdentifier(c); c = nextChar()) { + shiftFn(); + for (c = peekFn(); continuesIdentifier(c); c = peekFn()) { keyword += c; + shiftFn(); } if (auto search = keywords.find(keyword); search != keywords.end()) { + // There has been one more call to `peekFn` than to `shiftFn`. + // If they are optimized "quick" functions for `ViewedContent`, + // instead of `peek` and `shiftChar`, they have not updated + // `lexerState->expansionScanDistance`, so it must be incremented + // if it was previously zero. + if (Quick && lexerState->expansionScanDistance == 0) { + ++lexerState->expansionScanDistance; + } return Token(search->second); } } } - if (int c = bumpChar(); c == EOF) { + shiftFn(); + if (c == EOF) { return Token(T_(YYEOF)); } else if (isNewline(c)) { - handleCRLF(c); - nextLine(); + // Like `handleCRLF` but calling generic `shiftFn` + if (c == '\r' && peekFn() == '\n') { + shiftFn(); + } + nextLineFn(); lexerState->atLineStart = true; } } } +// This function is called when capturing `REPT`/`FOR` loops and `MACRO` bodies, +// and when skipping unexecuted `IF`/`ELIF`/`ELSE` blocks and `REPT`/`FOR` loops. +// It expects that these constructs' `ENDC`/`ENDR`/`ENDM` closing tokens are only +// valid at the start of their lines, which enables ignoring everything except +// the leading keyword in lines that have one (as well as line continuations). +// +// The only keywords it needs to recognize are case-insensitive `IF`, `ELIF`, +// `ELSE`, `ENDC`, `REPT`, `FOR`, `ENDR`, and `ENDM` (not `MACRO`). +// +// Note that when these constructs are *evaluated*, they can perform expansions +// (for macro args, interpolations, and macro invocations) which may produce +// tokens that would change how these constructs were captured or skipped, if +// they had been produced during the capture/skip non-evaluating phase. +static Token skipToLeadingKeyword() { + assume(!lexerState->enableExpansions); + + if (std::holds_alternative(lexerState->content) + && lexerState->expansionStack.empty()) { + // Optimize the common case (a fully-read assembly file without ongoing + // expansions) to avoid the bookkeeping of `peek` and `shiftChar`. + auto &view = std::get(lexerState->content); + char const *ptr = view.span.ptr.get(); + auto quickPeek = [&]() { return view.offset < view.span.size ? ptr[view.offset] : EOF; }; + auto quickNextLine = []() { ++lexerState->lineNo; }; + if (lexerState->capturing) { + assume(lexerState->captureBuf == nullptr); + auto quickCaptureShiftChar = [&]() { + ++view.offset; + ++lexerState->captureSize; + }; + return skipToLeadingKeyword(quickPeek, quickCaptureShiftChar, quickNextLine); + } else { + auto quickShiftChar = [&]() { ++view.offset; }; + return skipToLeadingKeyword(quickPeek, quickShiftChar, quickNextLine); + } + } else { + return skipToLeadingKeyword(peek, shiftChar, nextLine); + } +} + static Token skipIfBlock(bool toEndc) { lexer_SetMode(LEXER_NORMAL); From 23a77b6f656f5ee2881b5a748d55443ee0a3385d Mon Sep 17 00:00:00 2001 From: Rangi Date: Fri, 22 May 2026 18:33:19 -0400 Subject: [PATCH 2/3] Use concepts and rename `Quick` to `IsOptimized` --- src/asm/lexer.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/asm/lexer.cpp b/src/asm/lexer.cpp index f7be6a402..019a5d524 100644 --- a/src/asm/lexer.cpp +++ b/src/asm/lexer.cpp @@ -2082,8 +2082,10 @@ static Token yylex_RAW() { return Token(T_(YYEOF)); } -template -static Token skipToLeadingKeyword(PeekFnT peekFn, ShiftFnT shiftFn, NextLineFnT nextLineFn) { +template +static Token skipToLeadingKeyword( + InvocableR auto peekFn, Procedure<> auto shiftFn, Procedure<> auto nextLineFn +) { for (;;) { int c = peekFn(); if (lexerState->atLineStart) { @@ -2103,11 +2105,11 @@ static Token skipToLeadingKeyword(PeekFnT peekFn, ShiftFnT shiftFn, NextLineFnT } if (auto search = keywords.find(keyword); search != keywords.end()) { // There has been one more call to `peekFn` than to `shiftFn`. - // If they are optimized "quick" functions for `ViewedContent`, + // If they are optimized functions for unexpanded `ViewedContent`, // instead of `peek` and `shiftChar`, they have not updated // `lexerState->expansionScanDistance`, so it must be incremented // if it was previously zero. - if (Quick && lexerState->expansionScanDistance == 0) { + if (IsOptimized && lexerState->expansionScanDistance == 0) { ++lexerState->expansionScanDistance; } return Token(search->second); From 68832ee89ab1bfdede39d67b2524badda164a871 Mon Sep 17 00:00:00 2001 From: Rangi Date: Fri, 22 May 2026 18:47:37 -0400 Subject: [PATCH 3/3] Use a "finalize" callback instead of a Boolean flag --- src/asm/lexer.cpp | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/asm/lexer.cpp b/src/asm/lexer.cpp index 019a5d524..f051cfd4e 100644 --- a/src/asm/lexer.cpp +++ b/src/asm/lexer.cpp @@ -2082,9 +2082,11 @@ static Token yylex_RAW() { return Token(T_(YYEOF)); } -template static Token skipToLeadingKeyword( - InvocableR auto peekFn, Procedure<> auto shiftFn, Procedure<> auto nextLineFn + InvocableR auto peekFn, + Procedure<> auto shiftFn, + Procedure<> auto nextLineFn, + Procedure<> auto finalizeFn ) { for (;;) { int c = peekFn(); @@ -2104,14 +2106,7 @@ static Token skipToLeadingKeyword( shiftFn(); } if (auto search = keywords.find(keyword); search != keywords.end()) { - // There has been one more call to `peekFn` than to `shiftFn`. - // If they are optimized functions for unexpanded `ViewedContent`, - // instead of `peek` and `shiftChar`, they have not updated - // `lexerState->expansionScanDistance`, so it must be incremented - // if it was previously zero. - if (IsOptimized && lexerState->expansionScanDistance == 0) { - ++lexerState->expansionScanDistance; - } + finalizeFn(); return Token(search->second); } } @@ -2154,19 +2149,31 @@ static Token skipToLeadingKeyword() { char const *ptr = view.span.ptr.get(); auto quickPeek = [&]() { return view.offset < view.span.size ? ptr[view.offset] : EOF; }; auto quickNextLine = []() { ++lexerState->lineNo; }; + auto quickFinalize = []() { + // When `skipToLeadingKeyword` returns a token, there has been one more + // call to `quickPeek` than to `quickNextLine`. Unlike `peek` and `shiftChar`, + // the optimized functions do not update `lexerState->expansionScanDistance`, + // so it must be incrementedif it was previously zero. + if (lexerState->expansionScanDistance == 0) { + ++lexerState->expansionScanDistance; + } + }; if (lexerState->capturing) { assume(lexerState->captureBuf == nullptr); auto quickCaptureShiftChar = [&]() { ++view.offset; ++lexerState->captureSize; }; - return skipToLeadingKeyword(quickPeek, quickCaptureShiftChar, quickNextLine); + return skipToLeadingKeyword( + quickPeek, quickCaptureShiftChar, quickNextLine, quickFinalize + ); } else { auto quickShiftChar = [&]() { ++view.offset; }; - return skipToLeadingKeyword(quickPeek, quickShiftChar, quickNextLine); + return skipToLeadingKeyword(quickPeek, quickShiftChar, quickNextLine, quickFinalize); } } else { - return skipToLeadingKeyword(peek, shiftChar, nextLine); + auto finalize = []() {}; + return skipToLeadingKeyword(peek, shiftChar, nextLine, finalize); } }