diff --git a/Sources/SwiftParser/Parser.swift b/Sources/SwiftParser/Parser.swift index f5a90f6fc29..db154461f41 100644 --- a/Sources/SwiftParser/Parser.swift +++ b/Sources/SwiftParser/Parser.swift @@ -239,20 +239,23 @@ public struct Parser { self.arena = arena ?? ParsingRawSyntaxArena(parseTriviaFunction: TriviaParser.parseTrivia) var input = input - if copySource { - // Copy the source into a parser-owned buffer so the caller may free their - // buffer once this initializer returns. The buffer is freed when this - // `Parser` is destroyed (i.e. when the parse call returns); each token's - // text is interned into the arena at node-creation time, so the resulting - // tree does not reference this buffer. + if parseTransition == nil { + // Full parse: copy the whole source into the arena and lex over that copy, + // so parsed tokens point into arena-owned memory and need no per-token + // interning. + input = self.arena.internSourceBuffer(input) + self.sourceBufferOwner = nil + } else if copySource { + // Incremental reparse over a buffer that will not outlive this + // initializer: keep a parser-owned copy, freed when this `Parser` is + // destroyed. Each re-lexed token's text is interned into the arena. let owner = ParserSourceBufferOwner(copying: input) self.sourceBufferOwner = owner input = owner.buffer } else { - // No-copy: lex directly over the caller-provided buffer. The caller - // guarantees it stays valid for the entire parse (see `withParser`). Each - // token's text is interned into the arena, so the tree is self-contained - // and does not reference the buffer after parsing. + // Incremental reparse over the caller-provided buffer, which the caller + // guarantees stays valid for the parse (see `withParser`). Each re-lexed + // token's text is interned into the arena. self.sourceBufferOwner = nil } diff --git a/Sources/SwiftSyntax/Raw/RawSyntaxArena.swift b/Sources/SwiftSyntax/Raw/RawSyntaxArena.swift index c6d28b41428..038b829c44d 100644 --- a/Sources/SwiftSyntax/Raw/RawSyntaxArena.swift +++ b/Sources/SwiftSyntax/Raw/RawSyntaxArena.swift @@ -202,18 +202,57 @@ public class ParsingRawSyntaxArena: RawSyntaxArena { /// - Important: Must never be changed to a mutable value. See `RawSyntaxArenaRef.parseTrivia`. private let parseTriviaFunction: ParseTriviaFunction + /// The arena's own copy of the whole source buffer, made by + /// `internSourceBuffer` for a full parse. `nil` for an incremental reparse, + /// which does not copy the whole buffer. When set, the lexer runs over this + /// copy, so a parsed token whose text lies within it is already arena-owned + /// and `internParsedTokenText` returns it unchanged. + private struct SourceCopy { + let start: UnsafePointer + let end: UnsafePointer + } + private var sourceCopy: SourceCopy? + public init(parseTriviaFunction: @escaping ParseTriviaFunction) { self.parseTriviaFunction = parseTriviaFunction super.init(slabSize: 4096) } - /// Intern a parsed token's whole text into the arena's node allocator so the - /// resulting node does not depend on the source buffer. + /// Copy the whole source buffer into this arena and return the copy, so the + /// caller can lex over arena-owned memory. Parsed tokens then point directly + /// into the copy and need no per-token interning, and the resulting tree is + /// self-contained without referencing the caller's buffer. + /// + /// This suits a full parse, where every token's text is needed and the tree + /// legitimately retains text the size of the source. It must not be used for + /// an incremental reparse, where copying the whole buffer would defeat the + /// point of reusing nodes; there, `internParsedTokenText` copies each + /// re-lexed token individually instead. + public func internSourceBuffer(_ buffer: UnsafeBufferPointer) -> UnsafeBufferPointer { + guard buffer.count > 0 else { + self.sourceCopy = nil + return buffer + } + let dest = allocateTextBuffer(count: buffer.count) + _ = dest.initialize(from: buffer) + let copyBase = UnsafePointer(dest.baseAddress!) + self.sourceCopy = SourceCopy(start: copyBase, end: copyBase + buffer.count) + return UnsafeBufferPointer(dest) + } + + /// Intern a parsed token's whole text so the resulting node owns its text and + /// does not depend on the caller's source buffer. /// - /// Unlike `intern(_:)`, this skips the `contains` check: lexer-produced text - /// is never already managed by the arena, so a copy is always needed. + /// For a full parse the source was copied into the arena up front + /// (`internSourceBuffer`), so a token whose text lies within that copy is + /// already arena-owned and is returned unchanged. Otherwise — an incremental + /// reparse, or text that is not part of the source buffer such as a + /// synthesized token — the text is copied into the arena. func internParsedTokenText(_ text: SyntaxText) -> SyntaxText { - if text.isEmpty { + guard let base = text.baseAddress, !text.isEmpty else { + return text + } + if let sourceCopy, base >= sourceCopy.start, base + text.count <= sourceCopy.end { return text } let allocated = allocateTextBuffer(count: text.count)