[SwiftParser] Intern source text per parsed token

Previously, parsing interned the entire source buffer into the
`RawSyntaxArena`, so a parsed token's `wholeText` pointed into it and
the whole buffer stayed resident for the lifetime of the tree. For
incremental reparsing this is wasteful: each reparse allocates a fresh
arena that copies the full source again, even though most nodes are
reused from the previous tree.

Instead, intern each parsed token's text into the node allocator at
token creation. The resulting tree is self-contained and no longer
references the source buffer, so the buffer only needs to stay valid for
the duration of the parse. The static `parse`/`parseIncrementally`
methods, string-interpolation literal construction, `NameMatcher`, and
the plugin macro-expansion host now lex directly over the caller's
buffer via the new no-copy `Parser.withParser` entry points, copying
nothing. The `Parser(_:)` initializers, whose parse happens after the
initializer returns, instead keep a `Parser`-owned copy of the source
that is freed when the `Parser` is destroyed.

The source buffer is no longer null-terminated; all lexer reads are
bounds-checked against the buffer length, so the trailing NUL was
unnecessary. Add `testParseBufferEOFEdgeCases` covering EOF-adjacent
lexing (unterminated comments/strings, truncated UTF-8, etc.) over
exactly-sized allocations so the absence of the terminator is verified
under AddressSanitizer.

Author: rintaro

Sources/SwiftCompilerPluginMessageHandling/PluginMacroExpansionContext.swift

@@ -47,26 +47,29 @@ class ParsedSyntaxRegistry {
     swiftVersion: Parser.SwiftVersion,
     experimentalFeatures: Parser.ExperimentalFeatures
   ) -> Syntax {
-    var parser = Parser(
-      source,
+    // The parse runs entirely within `withParser`, so lex directly over the
+    // source without copying it into a parser-owned buffer.
+    return Parser.withParser(
+      source: source,
       swiftVersion: swiftVersion,
       experimentalFeatures: experimentalFeatures
-    )
-    switch kind {
-    case .declaration:
-      return Syntax(DeclSyntax.parse(from: &parser))
-    case .statement:
-      return Syntax(StmtSyntax.parse(from: &parser))
-    case .expression:
-      return Syntax(ExprSyntax.parse(from: &parser))
-    case .type:
-      return Syntax(TypeSyntax.parse(from: &parser))
-    case .pattern:
-      return Syntax(PatternSyntax.parse(from: &parser))
-    case .attribute:
-      return Syntax(AttributeSyntax.parse(from: &parser))
-    case .accessor:
-      return Syntax(AccessorDeclSyntax.parse(from: &parser))
+    ) { parser in
+      switch kind {
+      case .declaration:
+        return Syntax(DeclSyntax.parse(from: &parser))
+      case .statement:
+        return Syntax(StmtSyntax.parse(from: &parser))
+      case .expression:
+        return Syntax(ExprSyntax.parse(from: &parser))
+      case .type:
+        return Syntax(TypeSyntax.parse(from: &parser))
+      case .pattern:
+        return Syntax(PatternSyntax.parse(from: &parser))
+      case .attribute:
+        return Syntax(AttributeSyntax.parse(from: &parser))
+      case .accessor:
+        return Syntax(AccessorDeclSyntax.parse(from: &parser))
+      }
     }
   }
 

Sources/SwiftIDEUtils/NameMatcher.swift

@@ -11,10 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 #if compiler(>=6)
-import SwiftParser
+@_spi(RawSyntax) import SwiftParser
 public import SwiftSyntax
 #else
-import SwiftParser
+@_spi(RawSyntax) import SwiftParser
 import SwiftSyntax
 #endif
 
@@ -227,8 +227,9 @@ public class NameMatcher: SyntaxAnyVisitor {
       let triviaRangeEndOffsetInToken = triviaRange.upperBound.utf8Offset - token.position.utf8Offset
       let commentTree = token.syntaxTextBytes[positionOffsetInToken..<triviaRangeEndOffsetInToken]
         .withUnsafeBufferPointer { (buffer) -> ExprSyntax in
-          var parser = Parser(buffer)
-          return ExprSyntax.parse(from: &parser)
+          // The parse runs entirely within this scope, so lex directly over
+          // `buffer` without copying it.
+          Parser.withParser(source: buffer) { ExprSyntax.parse(from: &$0) }
         }
       // Run a new `NameMatcher`. Since the input of that name matcher is the text after the position to resolve, we
       // want to resolve the position at offset 0.

Sources/SwiftParser/ParseSourceFile.swift

@@ -22,8 +22,13 @@ extension Parser {
   public static func parse(
     source: String
   ) -> SourceFileSyntax {
-    var parser = Parser(source)
-    return SourceFileSyntax.parse(from: &parser)
+    return withParser(
+      source: source,
+      maximumNestingLevel: nil,
+      parseTransition: nil,
+      swiftVersion: nil,
+      experimentalFeatures: []
+    ) { SourceFileSyntax.parse(from: &$0) }
   }
 
   /// A compiler interface that allows the enabling of experimental features.
@@ -33,8 +38,13 @@ extension Parser {
     swiftVersion: SwiftVersion? = nil,
     experimentalFeatures: ExperimentalFeatures
   ) -> SourceFileSyntax {
-    var parser = Parser(source, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures)
-    return SourceFileSyntax.parse(from: &parser)
+    return withParser(
+      source: source,
+      maximumNestingLevel: nil,
+      parseTransition: nil,
+      swiftVersion: swiftVersion,
+      experimentalFeatures: experimentalFeatures
+    ) { SourceFileSyntax.parse(from: &$0) }
   }
 
   /// Parse the source code in the given buffer as Swift source file. See
@@ -44,8 +54,13 @@ extension Parser {
     maximumNestingLevel: Int? = nil,
     swiftVersion: SwiftVersion? = nil
   ) -> SourceFileSyntax {
-    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel, swiftVersion: swiftVersion)
-    return SourceFileSyntax.parse(from: &parser)
+    return withParser(
+      source: source,
+      maximumNestingLevel: maximumNestingLevel,
+      parseTransition: nil,
+      swiftVersion: swiftVersion,
+      experimentalFeatures: []
+    ) { SourceFileSyntax.parse(from: &$0) }
   }
 
   /// Parse the source code in the given string as Swift source file with support
@@ -120,8 +135,13 @@ extension Parser {
     source: String,
     parseTransition: IncrementalParseTransition?
   ) -> IncrementalParseResult {
-    var parser = Parser(source, parseTransition: parseTransition)
-    return IncrementalParseResult(tree: SourceFileSyntax.parse(from: &parser), lookaheadRanges: parser.lookaheadRanges)
+    return withParser(
+      source: source,
+      maximumNestingLevel: nil,
+      parseTransition: parseTransition,
+      swiftVersion: nil,
+      experimentalFeatures: []
+    ) { IncrementalParseResult(tree: SourceFileSyntax.parse(from: &$0), lookaheadRanges: $0.lookaheadRanges) }
   }
 
   /// Parse the source code in the given buffer as Swift source file with support
@@ -133,8 +153,13 @@ extension Parser {
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition?
   ) -> IncrementalParseResult {
-    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition)
-    return IncrementalParseResult(tree: SourceFileSyntax.parse(from: &parser), lookaheadRanges: parser.lookaheadRanges)
+    return withParser(
+      source: source,
+      maximumNestingLevel: maximumNestingLevel,
+      parseTransition: parseTransition,
+      swiftVersion: nil,
+      experimentalFeatures: []
+    ) { IncrementalParseResult(tree: SourceFileSyntax.parse(from: &$0), lookaheadRanges: $0.lookaheadRanges) }
   }
 }