@@ -149,6 +149,9 @@ interface GrammarJsContext {
149149 * `template_chars` token. `null` when no template token exists.
150150 */
151151 templatePlan : TemplatePlan | null ;
152+ /** String tokens carrying highlight-only interpolation regions, each re-expressed as a rule
153+ * backed by an external `<rule>_chars` token (parallel to `templatePlan`). Empty if none. */
154+ interpolationPlans : InterpolationPlan [ ] ;
152155 /**
153156 * Ref nodes (the identifier right after a definition keyword) that should be
154157 * wrapped in `field('name', …)` so highlights.scm can target them with the
@@ -358,6 +361,8 @@ function buildTokenBody(name: string, ctx: GrammarJsContext): string | null {
358361 // The interpolated-template token is re-expressed as a `template` RULE (with
359362 // `${ … }` holes that re-enter the expression grammar), emitted separately.
360363 if ( ctx . templatePlan && ctx . templatePlan . tokenName === name ) return null ;
364+ // A string token with interpolation regions is likewise re-expressed as a rule (emitted separately).
365+ if ( ctx . interpolationPlans . some ( ip => ip . tokenName === name ) ) return null ;
361366 // Skip-flagged tokens (comments, whitespace) go in `extras`, not as a named
362367 // rule reference — but we still emit them so highlights can capture comments.
363368 // tree-sitter's token() DFA rejects zero-width assertions, so strip them first.
@@ -538,6 +543,50 @@ function planTemplate(grammar: CstGrammar): TemplatePlan | null {
538543 } ;
539544}
540545
546+ /**
547+ * A string token carrying highlight-only interpolation regions (e.g. env-spec `${…}` / `$(…)`),
548+ * re-expressed as a tree-sitter RULE (open delim + chars/interpolation runs + close delim) — the
549+ * same shape a template literal gets. The literal text between regions is an external
550+ * `<rule>_chars` token (the scanner stops it at the close delim or any region opener).
551+ */
552+ interface InterpolationPlan {
553+ tokenName : string ; // original token name (e.g. 'DQ') — now emitted as a rule, not a token
554+ ruleSnake : string ; // snake rule name (e.g. 'dq') — keeps `$.dq` references valid
555+ charsSnake : string ; // external scanner symbol for the literal text (e.g. 'dq_chars')
556+ open : string ; // opening delimiter (e.g. '"')
557+ close : string ; // closing delimiter (same as open for a string token)
558+ regions : { ruleSnake : string ; open : string ; close : string } [ ] ; // one sub-rule per interpolation entry
559+ }
560+
561+ // Decode an author-supplied interpolation begin/end REGEX fragment to the literal text it matches:
562+ // drop an optional leading escaped-backslash (`\\?`, the env-spec `\${` vs `${` allowance), then
563+ // unescape the rest. Targets the scanner-friendly forms (decoded literal length 1–2; see PR #9).
564+ function decodeInterpDelim ( src : string ) : string {
565+ return src . replace ( / \\ \\ \? / g, '' ) . replace ( / \\ ( .) / g, '$1' ) ;
566+ }
567+
568+ function planInterpolations ( grammar : CstGrammar ) : InterpolationPlan [ ] {
569+ const plans : InterpolationPlan [ ] = [ ] ;
570+ for ( const tok of grammar . tokens ) {
571+ if ( ! tok . interpolation ?. length ) continue ;
572+ const open = tokenPatternStringDelimiters ( tok ) [ 0 ] ?? '"' ;
573+ const ruleSnake = toSnake ( tok . name ) ;
574+ plans . push ( {
575+ tokenName : tok . name ,
576+ ruleSnake,
577+ charsSnake : ruleSnake + '_chars' ,
578+ open,
579+ close : open ,
580+ regions : tok . interpolation . map ( ( interp , i ) => ( {
581+ ruleSnake : `${ ruleSnake } ${ i + 1 } ,
582+ open : decodeInterpDelim ( interp . begin ) ,
583+ close : decodeInterpDelim ( interp . end ) ,
584+ } ) ) ,
585+ } ) ;
586+ }
587+ return plans ;
588+ }
589+
541590/** Determine which tokens the external scanner must provide. */
542591function planScannerTokens ( grammar : CstGrammar ) : Map < string , string > {
543592 const map = new Map < string , string > ( ) ;
@@ -560,6 +609,7 @@ function planScannerTokens(grammar: CstGrammar): Map<string, string> {
560609function externalSymbols ( ctx : GrammarJsContext ) : string [ ] {
561610 const syms = [ ...ctx . scannerTokenFor . values ( ) ] ;
562611 if ( ctx . templatePlan ) syms . push ( ctx . templatePlan . charsSnake ) ;
612+ for ( const ip of ctx . interpolationPlans ) syms . push ( ip . charsSnake ) ;
563613 return syms ;
564614}
565615
@@ -725,8 +775,10 @@ export function generateTreeSitter(grammar: CstGrammar, langName?: string): Tree
725775
726776 const scannerTokenFor = planScannerTokens ( grammar ) ;
727777 const templatePlan = planTemplate ( grammar ) ;
778+ const interpolationPlans = planInterpolations ( grammar ) ;
728779 const externalSnake = new Set ( [ ...scannerTokenFor . values ( ) ] ) ;
729780 if ( templatePlan ) externalSnake . add ( templatePlan . charsSnake ) ;
781+ for ( const ip of interpolationPlans ) externalSnake . add ( ip . charsSnake ) ;
730782
731783 // Find the identifier nodes that follow a declaration keyword, so we can wrap
732784 // them in `field('name', …)` in grammar.js AND emit standard `name:` highlight
@@ -736,6 +788,7 @@ export function generateTreeSitter(grammar: CstGrammar, langName?: string): Tree
736788 const ctx : GrammarJsContext = {
737789 grammar, tokenNames, ruleSnake, tokenSnake, prattRules, externalSnake, scannerTokenFor,
738790 templatePlan,
791+ interpolationPlans,
739792 nameFieldNodes : nameFields . nodes ,
740793 } ;
741794
@@ -859,6 +912,27 @@ function buildGrammarJs(ctx: GrammarJsContext, grammarName: string): string {
859912 ) ;
860913 }
861914
915+ // String-interpolation tokens: re-expressed as a rule (open + chars/interpolation runs + close);
916+ // each interpolation region is a sub-rule whose hole re-enters the expression grammar (like a template).
917+ const interpExprName = [ ...ctx . prattRules ] [ 0 ] ;
918+ const interpExprSnake = interpExprName ? ctx . ruleSnake . get ( interpExprName ) ! : null ;
919+ const interpHole = interpExprSnake ? `optional($.${ interpExprSnake } : 'blank()' ;
920+ for ( const ip of ctx . interpolationPlans ) {
921+ const choices = [ `$.${ ip . charsSnake } , ...ip . regions . map ( r => `$.${ r . ruleSnake } ) ] . join ( ', ' ) ;
922+ ruleEntries . push (
923+ ` ${ ip . ruleSnake } +
924+ ` ${ jsString ( ip . open ) } +
925+ ` repeat(choice(${ choices } +
926+ ` ${ jsString ( ip . close ) } +
927+ ` )` ,
928+ ) ;
929+ for ( const r of ip . regions ) {
930+ ruleEntries . push (
931+ ` ${ r . ruleSnake } ${ jsString ( r . open ) } ${ interpHole } ${ jsString ( r . close ) } ,
932+ ) ;
933+ }
934+ }
935+
862936 lines . push ( ruleEntries . join ( ',\n\n' ) ) ;
863937 lines . push ( ' }' ) ;
864938 lines . push ( '});' ) ;
@@ -1087,6 +1161,15 @@ function buildHighlightsScm(
10871161 tokenNodeCaptures . push ( { query : `(${ tpl . substRuleSnake } ${ jsString ( tpl . interpOpen ) } , capture : '@punctuation.special' } ) ;
10881162 tokenNodeCaptures . push ( { query : `(${ tpl . substRuleSnake } ${ jsString ( tpl . interpClose ) } , capture : '@punctuation.special' } ) ;
10891163 }
1164+ // String-interpolation regions: the literal text reads as string; the region delimiters as
1165+ // punctuation — same treatment as a template hole, derived from the interpolation metadata.
1166+ for ( const ip of ctx . interpolationPlans ) {
1167+ tokenNodeCaptures . push ( { query : `(${ ip . charsSnake } , capture : '@string' } ) ;
1168+ for ( const r of ip . regions ) {
1169+ tokenNodeCaptures . push ( { query : `(${ r . ruleSnake } ${ jsString ( r . open ) } , capture : '@punctuation.special' } ) ;
1170+ tokenNodeCaptures . push ( { query : `(${ r . ruleSnake } ${ jsString ( r . close ) } , capture : '@punctuation.special' } ) ;
1171+ }
1172+ }
10901173
10911174 // ── D. Contextual node captures via emitted fields ──
10921175 // Operators carry an `operator` field in Pratt rules; they're already covered by
@@ -1753,6 +1836,49 @@ function buildScannerC(
17531836 L . push ( '' ) ;
17541837 }
17551838
1839+ // ── Interpolated-string char scanners (one per string token carrying interpolation) ──
1840+ // Each scans the literal run inside the string, stopping before the close delimiter or any
1841+ // interpolation opener (so the opener re-enters the expression grammar via its sub-rule). The
1842+ // openers are DATA from the interpolation metadata (decoded literals, length 1–2).
1843+ {
1844+ const cChar = ( ch : string ) => ch === '\\' ? "'\\\\'" : ch === "'" ? "'\\''" : `'${ ch } ;
1845+ for ( const ip of ctx . interpolationPlans ) {
1846+ const charsSym = ip . charsSnake . toUpperCase ( ) ;
1847+ const up = ip . ruleSnake . toUpperCase ( ) ;
1848+ const openerInit = ip . regions . map ( r => jsString ( r . open ) ) . join ( ', ' ) ;
1849+ L . push ( `// ── Interpolated-string scan (${ ip . tokenName } ) ;
1850+ L . push ( `static const char *${ up } ${ openerInit } ) ;
1851+ L . push ( `static const unsigned ${ up } ${ ip . regions . length } ) ;
1852+ L . push ( `static bool scan_${ ip . ruleSnake } ) ;
1853+ L . push ( ' bool has_content = false;' ) ;
1854+ L . push ( ' for (;;) {' ) ;
1855+ L . push ( ' lexer->mark_end(lexer);' ) ;
1856+ L . push ( ' int32_t c = lexer->lookahead;' ) ;
1857+ L . push ( ' if (c == 0) return false; // EOF — let the CFG report the unterminated string' ) ;
1858+ L . push ( ` if (c == ${ cChar ( ip . close ) } ) ;
1859+ L . push ( ' bool first_match = false;' ) ;
1860+ L . push ( ` for (unsigned i = 0; i < ${ up } ${ up } ) ;
1861+ L . push ( ' if (first_match) {' ) ;
1862+ L . push ( ' advance(lexer); // peek past the opener\'s first char' ) ;
1863+ L . push ( ' int32_t c2 = lexer->lookahead;' ) ;
1864+ L . push ( ' bool real = false;' ) ;
1865+ L . push ( ` for (unsigned i = 0; i < ${ up } ) ;
1866+ L . push ( ` if ((int32_t)${ up } ${ up } ${ up } ) ;
1867+ L . push ( ' if (real) break; // a real opener — token ends before it (mark_end frozen above)' ) ;
1868+ L . push ( ' has_content = true; continue; // lone first char → literal content' ) ;
1869+ L . push ( ' }' ) ;
1870+ L . push ( ' if (c == \'\\\\\') { advance(lexer); if (lexer->lookahead != 0) advance(lexer); has_content = true; continue; }' ) ;
1871+ L . push ( ' advance(lexer);' ) ;
1872+ L . push ( ' has_content = true;' ) ;
1873+ L . push ( ' }' ) ;
1874+ L . push ( ' if (!has_content) return false;' ) ;
1875+ L . push ( ` lexer->result_symbol = ${ charsSym } ) ;
1876+ L . push ( ' return true;' ) ;
1877+ L . push ( '}' ) ;
1878+ L . push ( '' ) ;
1879+ }
1880+ }
1881+
17561882 // ── scan() entry ──
17571883 L . push ( 'bool tree_sitter_' + grammarName + '_external_scanner_scan(void *payload, TSLexer *lexer,' ) ;
17581884 L . push ( ' const bool *valid_symbols) {' ) ;
@@ -1797,6 +1923,14 @@ function buildScannerC(
17971923 L . push ( ' }' ) ;
17981924 L . push ( '' ) ;
17991925 }
1926+ for ( const ip of ctx . interpolationPlans ) {
1927+ const charsSym = ip . charsSnake . toUpperCase ( ) ;
1928+ L . push ( ` // ${ ip . tokenName } ) ;
1929+ L . push ( ` if (valid_symbols[${ charsSym } ) ;
1930+ L . push ( ` if (scan_${ ip . ruleSnake } ) ;
1931+ L . push ( ' }' ) ;
1932+ L . push ( '' ) ;
1933+ }
18001934 L . push ( ' return false;' ) ;
18011935 L . push ( '}' ) ;
18021936 L . push ( '' ) ;
0 commit comments