diff --git a/Sources/SwiftParser/Lexer/Cursor.swift b/Sources/SwiftParser/Lexer/Cursor.swift index 007b7894270..cdb6305c959 100644 --- a/Sources/SwiftParser/Lexer/Cursor.swift +++ b/Sources/SwiftParser/Lexer/Cursor.swift @@ -255,6 +255,28 @@ extension Lexer { } var position: Position + /// Says which mode we are in (pure Swift versus some literate syntax). + var mode: Parser.Mode + + /// `true` if we are inside a code block in a literate Swift file. + var inCodeBlock: Bool + + /// `true` if we are scanning indentation. + var scanningIndent: Bool + + /// The current indent level, for literate parsing. + var indentLevel: Int + + /// The current Markdown fence character (either "`" or " ") + var markdownFence: Unicode.Scalar? + + /// The length of the Markdown fence + var markdownFenceLength: Int + + /// The current reStructuredText indent level, for literate parsing. + var rstIndentLevel: Int + + /// Says which experimental features are enabled. var experimentalFeatures: Parser.ExperimentalFeatures /// If we have already lexed a token, the kind of the previously lexed token @@ -269,9 +291,21 @@ extension Lexer { private var stateStack: StateStack = StateStack() - init(input: UnsafeBufferPointer, previous: UInt8, experimentalFeatures: Parser.ExperimentalFeatures) { + init( + input: UnsafeBufferPointer, + previous: UInt8, + mode: Parser.Mode = .swift, + experimentalFeatures: Parser.ExperimentalFeatures + ) { self.position = Position(input: input, previous: previous) + self.mode = mode self.experimentalFeatures = experimentalFeatures + self.inCodeBlock = self.mode == .swift + self.indentLevel = 0 + self.scanningIndent = true + self.rstIndentLevel = 0 + self.markdownFence = nil + self.markdownFenceLength = 0 } /// Returns `true` if this cursor is sufficiently different to `other` in a way that indicates that the lexer has @@ -305,6 +339,9 @@ extension Lexer { func distance(to other: Self) -> Int { self.position.distance(to: other.position) } + func distance(from other: Self) -> Int { + other.distance(to: self) + } var isAtEndOfFile: Bool { self.position.isAtEndOfFile @@ -428,18 +465,40 @@ extension Lexer.Cursor.Position { extension Lexer.Cursor { mutating func nextToken(sourceBufferStart: Lexer.Cursor, stateAllocator: BumpPtrAllocator) -> Lexer.Lexeme { - let cursor = self - // Leading trivia. + var cursor = self let leadingTriviaStart = self - let newlineInLeadingTrivia: NewlinePresence + var newlineInLeadingTrivia: NewlinePresence = .absent var diagnostic: TokenDiagnostic? = nil - if let leadingTriviaMode = self.currentState.leadingTriviaLexingMode(cursor: self) { - let triviaResult = self.lexTrivia(mode: leadingTriviaMode) - newlineInLeadingTrivia = triviaResult.newlinePresence - diagnostic = TokenDiagnostic(combining: diagnostic, triviaResult.error?.tokenDiagnostic(tokenStart: cursor)) - } else { - newlineInLeadingTrivia = .absent - } + + // Loop until we're in a code block + repeat { + cursor = self + + // In literate mode, if we aren't in a code block, find one + if !inCodeBlock { + self.advanceToCodeBlock(sourceBufferStart: sourceBufferStart) + + // We might have read the rest of the file + if self.isAtEndOfFile { + break + } + } + + if inCodeBlock { + // Leading trivia. + if let leadingTriviaMode = self.currentState.leadingTriviaLexingMode(cursor: self) { + let triviaResult = self.lexTrivia(mode: leadingTriviaMode) + newlineInLeadingTrivia = triviaResult.newlinePresence + diagnostic = TokenDiagnostic(combining: diagnostic, triviaResult.error?.tokenDiagnostic(tokenStart: cursor)) + } else { + newlineInLeadingTrivia = .absent + } + } + + if self.atEndOfCodeBlock() { + inCodeBlock = false + } + } while !inCodeBlock && !self.isAtEndOfFile // Token text. let textStart = self @@ -518,6 +577,568 @@ extension Lexer.Cursor { } +// MARK: - Literate Swift Processing + +extension Lexer.Cursor { + /// Move the cursor back to the start of the current line, if any + mutating func rewindToStartOfLine(sourceBufferStart: Lexer.Cursor) { + guard let bufferBaseAddress = sourceBufferStart.input.baseAddress, + var selfBaseAddress = self.input.baseAddress + else { + return + } + + let original = selfBaseAddress + var lastNonEOL = selfBaseAddress + while selfBaseAddress > bufferBaseAddress { + if selfBaseAddress.pointee == 13 || selfBaseAddress.pointee == 10 { + break + } + lastNonEOL = selfBaseAddress + selfBaseAddress -= 1 + } + + let count = self.input.count + (original - lastNonEOL) + self.position.input = UnsafeBufferPointer(start: lastNonEOL, count: count) + + if lastNonEOL > bufferBaseAddress { + self.position.previous = (lastNonEOL - 1).pointee + } else { + self.position.previous = 0 + } + } + + func previousLineWasEmpty(sourceBufferStart: Lexer.Cursor) -> Bool { + guard let bufferBaseAddress = sourceBufferStart.input.baseAddress, + var selfBaseAddress = self.input.baseAddress + else { + return true + } + + // Skip the preceding line ending + if selfBaseAddress > bufferBaseAddress && selfBaseAddress[-1] == "\n" { + selfBaseAddress -= 1 + } + if selfBaseAddress > bufferBaseAddress && selfBaseAddress[-1] == "\r" { + selfBaseAddress -= 1 + } + + // If we're at the start of the buffer, it was empty + if selfBaseAddress == bufferBaseAddress { + return true + } + + // If we're at a line ending, it was also empty + if selfBaseAddress > bufferBaseAddress + && (selfBaseAddress[-1] == "\n" || selfBaseAddress[-1] == "\r") + { + return true + } + + return false + } + + /// Move the cursor back to the line ending before the current line, if + /// any. + mutating func rewindToEndOfLine(sourceBufferStart: Lexer.Cursor) { + guard let bufferBaseAddress = sourceBufferStart.input.baseAddress, + var selfBaseAddress = self.input.baseAddress + else { + return + } + + let original = selfBaseAddress + while selfBaseAddress > bufferBaseAddress { + if selfBaseAddress.pointee == 13 || selfBaseAddress.pointee == 10 { + break + } + selfBaseAddress -= 1 + } + + // Skip backwards to the CR if we're at CR/LF + if selfBaseAddress > bufferBaseAddress && selfBaseAddress.pointee == 10 { + let prevAddress = selfBaseAddress - 1 + if prevAddress.pointee == 13 { + selfBaseAddress = prevAddress + } + } + + let count = self.input.count + (original - selfBaseAddress) + self.position.input = UnsafeBufferPointer(start: selfBaseAddress, count: count) + + if selfBaseAddress > bufferBaseAddress { + self.position.previous = (selfBaseAddress - 1).pointee + } else { + self.position.previous = 0 + } + } + + /// Skip the code in a reStructuredText code block + mutating func skipRSTCodeBlock() { + var lineStart = self + while true { + let pos = self + guard let ch = self.advance() else { + break + } + switch ch { + case 13: + lineStart = pos + _ = self.advance(matching: "\n") + indentLevel = 0; + case 10: + lineStart = pos + indentLevel = 0 + case 32: + indentLevel += 1 + case 9: + indentLevel += 8 + default: + if indentLevel <= rstIndentLevel { + self = lineStart + return + } + } + } + } + + /// Scan a word, which is a space-separated item in a literate file + mutating func scanWord(extraTerminator: CharacterByte? = nil) -> SyntaxText { + let start = self + let extraByte = extraTerminator?.value ?? 32 + self.advance(while: { + $0 != " " && $0 != "\t" + && $0 != "\r" && $0 != "\n" + && $0.value != extraByte + }) + + return start.text(upTo: self) + } + + enum IndentState { + case waitingForEOL + case gotCR + case gotEOL + case gotBlankLineCR + case gotBlankLine + } + + /// Find the start of the next Markdown code block + mutating func advanceToMarkdownCodeBlock(sourceBufferStart: Lexer.Cursor) { + var indentState = IndentState.gotEOL + indentLevel = 0 + + self.rewindToStartOfLine(sourceBufferStart: sourceBufferStart) + + // Looking for + // + // ``` swift nocompile + // + // or a block with a four plus character indent. + // + // If we see a non-Swift code block, we will skip it; we wil also skip + // any code block marked "nocompile". + markdownFence = nil + var fenceStart = self + while true { + let pos = self + guard let ch = self.advance() else { + break + } + if ch == "`" || ch == "~" { + if let mdf = markdownFence, ch == mdf { + let count = self.distance(from: fenceStart) + if count == 3 { + // Scan any additional fence characters and remember the length + self.advance(while: { $0 == mdf }) + + markdownFenceLength = self.distance(from: fenceStart) + + // Skip whitespace + self.advance(while: { $0 == " " || $0 == "\t" }) + + let terminator: CharacterByte = markdownFence == "`" ? "`" : " " + + // Scan a word + let language = scanWord(extraTerminator: terminator) + + var skip = language != "swift" && language != "" + + // Continue until end of line + while self.is( + notAt: "\r", + "\n", + markdownFence == "`" ? "`" : "\r" + ) { + // Skip whitespace + self.advance(while: { $0 == " " || $0 == "\t" }) + + let word = scanWord(extraTerminator: terminator) + if word == "nocompile" { + skip = true + } + } + + // If we're using backticks and found a backtick, this is not a + // code block; skip the backtick(s) and the rest of the line, and + // continue + if markdownFence == "`" && self.is(at: "`") { + self.advance(while: { $0 == "`" }) + + self.advanceToEndOfLine() + + markdownFence = nil + continue + } + + // If we're at a line end, we're done + if self.advanceMatchingEOL() { + if skip { + // Skip this block + var closeStart: Self? = nil + while true { + let pos = self + guard let ch = advance() else { break } + if ch == mdf { + if closeStart == nil { + closeStart = pos + } + if self.distance(from: closeStart!) == markdownFenceLength { + break + } + } else { + closeStart = nil + } + } + + if self.distance(from: closeStart!) == markdownFenceLength { + self.advance(while: { $0 == mdf }) + } + markdownFence = nil + continue + } else { + inCodeBlock = true + return + } + } + } + } else { + markdownFence = Unicode.Scalar(ch) + fenceStart = pos + } + } else { + markdownFence = nil + } + + switch indentState { + case .waitingForEOL: + if ch == "\r" { + indentState = .gotCR + } else if ch == "\n" { + indentState = .gotEOL + } + case .gotCR: + if ch == "\n" { + indentState = .gotEOL + break + } + indentState = .gotEOL + fallthrough + case .gotEOL: + if ch == "\r" { + indentState = .gotBlankLineCR + } else if ch == "\n" { + indentState = .gotBlankLine + indentLevel = 0 + } else if ch != " " && ch != "\t" { + indentState = .waitingForEOL + } + break + case .gotBlankLineCR: + if ch == "\n" { + indentState = .gotBlankLine + indentLevel = 0 + break + } + indentState = .gotBlankLine + fallthrough + case .gotBlankLine: + if ch == " " { + indentLevel += 1 + } else if ch == "\t" { + indentLevel += 8 + } else { + indentState = .waitingForEOL + } + } + + // If we have a four plus space indent, this is a code block + if indentLevel >= 4 { + scanningIndent = false + markdownFence = " " + inCodeBlock = true + return + } + } + } + + // Scan an EOL + mutating func advanceMatchingEOL() -> Bool { + if self.is(notAt: "\r", "\n") { + return false + } + _ = self.advance(matching: "\r") + _ = self.advance(matching: "\n") + return true + } + + // Skip over whitespace + mutating func advanceOverWhitespace() { + self.advance(while: { $0 == " " || $0 == "\t" }) + } + + // Find the start of the next reStructuredText code block + mutating func advanceToRSTCodeBlock(sourceBufferStart: Lexer.Cursor) { + var indentState: IndentState = .gotEOL + rstIndentLevel = 0 + indentLevel = 0 + + self.rewindToStartOfLine(sourceBufferStart: sourceBufferStart) + + if self.previousLineWasEmpty(sourceBufferStart: sourceBufferStart) { + indentState = .gotBlankLine + } + + while let ch = self.advance() { + // Looking for :: EOL WS* EOL + if ch == ":" && self.is(at: ":") { + _ = self.advance() + + if !self.advanceMatchingEOL() { + continue + } + + self.advanceOverWhitespace() + + if !self.advanceMatchingEOL() { + continue + } + + rstIndentLevel = indentLevel + inCodeBlock = true + return + } + + switch indentState { + case .waitingForEOL: + if ch == "\r" { + indentState = .gotCR + } else if ch == "\n" { + indentState = .gotEOL + } + case .gotCR: + if ch == "\n" { + indentState = .gotEOL + break + } + indentState = .gotEOL + fallthrough + case .gotEOL: + if ch == "\r" { + indentState = .gotBlankLineCR + } else if ch == "\n" { + indentState = .gotBlankLine + indentLevel = 0 + } else if ch != " " || ch != "\t" { + indentState = .waitingForEOL + } + case .gotBlankLineCR: + if ch == "\n" { + indentState = .gotBlankLine + indentLevel = 0 + break + } + indentState = .gotBlankLine + fallthrough + case .gotBlankLine: + if ch == " " { + indentLevel += 1 + } else if ch == "\t" { + indentLevel += 8 + } else if ch == "\r" { + indentState = .gotBlankLineCR + indentLevel = 0 + } else if ch == "\n" { + indentState = .gotBlankLine + indentLevel = 0 + } else if (ch == "-" || ch == "*" || ch == "+") + && self.is(at: " ", "\t") + { + // It's a bullet + indentLevel += 1 + var extraIndent = 0 + self.advance(while: { + if $0 == " " { + extraIndent += 1 + return true + } + if $0 == "\t" { + extraIndent += 8 + return true + } + return false + }) + indentLevel += extraIndent + } else if (ch >= UInt8(ascii: "0") && ch <= UInt8(ascii: "9")) || ch == "#" { + let itemPos = self + + indentLevel += 1 + if ch != "#" { + self.advance(while: { $0.value >= UInt8(ascii: "0") && $0.value <= UInt8(ascii: "9") }) + indentLevel += self.distance(from: itemPos) + } + + if self.is(notAt: ".") { + self = itemPos + indentState = .waitingForEOL + break + } + + // This is an enumerated list item + var extraIndent = 0 + self.advance(while: { + if $0 == " " { + extraIndent += 1 + return true + } + if $0 == "\t" { + extraIndent += 8 + return true + } + return false + }) + indentLevel += extraIndent + } else if ch == "." && self.is(at: ". ") { + // It's explicit markup + _ = self.advance() + self.advance(while: { $0 == " " }) + + if self.is(at: "code-block::", "sourcecode::") { + self.advance(by: 12) + + // We're in a code-block:: directive + self.advance(while: { $0 == " " || $0 == "\t" }) + + // Scan the language + let language = scanWord() + + var skip = language != "swift" && language != "" + + // Scan to the next line + self.advanceToStartOfLine() + + // Process options + while let ch = self.advance() { + if ch == " " || ch == "\t" { + continue + } + + if ch == ":" { + if self.is(at: "nocompile:") { + skip = true + } + + self.advanceToStartOfLine() + } else if ch == "\r" || ch == "\n" { + // We're at a code block + + rstIndentLevel = indentLevel + + if skip { + self.skipRSTCodeBlock() + break + } else { + inCodeBlock = true + return + } + } + } + } else { + self.advanceToEndOfLine() + } + + indentState = .waitingForEOL + } else { + indentState = .waitingForEOL + } + } + } + } + + // Find the start of the next LaTeX code block + mutating func advanceToLaTeXCodeBlock() { + while let ch = advance() { + if ch == "\\" { + if self.is(at: "begin{swift}") { + self.advanceToStartOfLine() + + inCodeBlock = true + return + } + } + } + } + + mutating func advanceToCodeBlock(sourceBufferStart: Lexer.Cursor) { + switch mode { + case .markdown: + advanceToMarkdownCodeBlock(sourceBufferStart: sourceBufferStart) + case .reStructuredText: + advanceToRSTCodeBlock(sourceBufferStart: sourceBufferStart) + case .laTeX: + advanceToLaTeXCodeBlock() + default: + fatalError("Unknown lexer mode in advanceToCodeBlock()") + } + + if inCodeBlock { + self.rewindToEndOfLine(sourceBufferStart: sourceBufferStart) + } + } + + mutating func atEndOfCodeBlock() -> Bool { + if !inCodeBlock { + return false + } + + switch mode { + case .markdown: + guard let markdownFence else { + return true + } + if markdownFence == " " { + return indentLevel < 4 + } + if self.is(notAt: CharacterByte(unicodeScalarLiteral: markdownFence)) { + return false + } + var pos = self + pos.advance(while: { $0 == markdownFence }) + if pos.distance(from: self) >= markdownFenceLength { + self = pos + return true + } + return false + case .reStructuredText: + return indentLevel <= rstIndentLevel + case .laTeX: + return self.is(at: "\\end{swift}") + default: + return false + } + } +} + // MARK: - Peeking /// Essentially a UInt8 that is expressible by integer and string literals. @@ -631,11 +1252,58 @@ extension Lexer.Cursor { return peeked != character1.value && peeked != character2.value && peeked != character3.value } + /// Returns `true` if we are not at the end of the file and the string at + /// offset `offset` is `text`. + @_disfavoredOverload + func `is`( + offset: Int = 0, + at text: StaticString + ) -> Bool { + if offset >= self.input.count { + return false + } + let length = text.utf8CodeUnitCount + if self.input.count - offset < length { + return false + } + + return self.text(upTo: self.advanced(by: length)) == SyntaxText(text) + } + + /// Returns `true` if we are not at the end of the file and the string at + /// offset `offset` is `text`. + @_disfavoredOverload + func `is`( + offset: Int = 0, + at text: StaticString, + _ text2: StaticString + ) -> Bool { + if offset >= self.input.count { + return false + } + + let remaining = self.input.count - offset + let textLen = text.utf8CodeUnitCount + if remaining >= textLen { + if self.text(upTo: self.advanced(by: textLen)) == SyntaxText(text) { + return true + } + } + let text2Len = text2.utf8CodeUnitCount + if remaining >= text2Len { + if self.text(upTo: self.advanced(by: text2Len)) == SyntaxText(text2) { + return true + } + } + + return false + } + // MARK: Misc /// Returns the text from `self` to `other`. func text(upTo other: Lexer.Cursor) -> SyntaxText { - let count = other.input.baseAddress! - self.input.baseAddress! + let count = self.distance(to: other) precondition(count >= 0) return SyntaxText(baseAddress: self.input.baseAddress, count: count) } @@ -673,6 +1341,18 @@ extension Lexer.Cursor { self.position.advance() } + /// Advance the cursor position by `n` bytes. + mutating func advance(by n: Int) { + self.position = self.position.advanced(by: n) + } + + /// Return a new cursor `n` bytes ahead of this one. + func advanced(by n: Int) -> Self { + var result = self + result.advance(by: n) + return result + } + /// If the current character is `matching`, advance the cursor and return `true`. /// Otherwise, this is a no-op and returns `false`. mutating func advance(matching: CharacterByte) -> Bool { @@ -739,6 +1419,13 @@ extension Lexer.Cursor { } } + /// Advance the cursor to the start of the next line. + mutating func advanceToStartOfLine() { + self.advanceToEndOfLine() + _ = self.advance(matching: "\r") + _ = self.advance(matching: "\n") + } + /// Returns `true` if the comment spanned multiple lines and `false` otherwise. /// Assumes that the curser is currently pointing at the `*` of the opening `/*`. mutating func advanceToEndOfSlashStarComment(slashPosition: Lexer.Cursor) -> TriviaResult { @@ -1231,23 +1918,36 @@ extension Lexer.Cursor { break } newlinePresence = .present + scanningIndent = true + indentLevel = 0 continue case "\r": if mode == .noNewlines { break } newlinePresence = .present + scanningIndent = true + indentLevel = 0 continue case " ": + if scanningIndent { + indentLevel += 1 + } continue case "\t": + if scanningIndent { + indentLevel += 8 + } continue case "\u{000B}": + scanningIndent = false continue case "\u{000C}": + scanningIndent = false continue case "/": + scanningIndent = false switch self.peek() { case "/": self.advanceToEndOfLine() @@ -1263,6 +1963,7 @@ extension Lexer.Cursor { break } case "<", ">": + scanningIndent = false if self.tryLexConflictMarker(start: start) { error = LexingDiagnostic(.sourceConflictMarker, position: start) continue @@ -1289,6 +1990,7 @@ extension Lexer.Cursor { // Start of operators. "%", "!", "?", "=", "-", "+", "*", "&", "|", "^", "~", ".": + scanningIndent = false break case 0xEF: if self.is(at: 0xBB), self.is(offset: 1, at: 0xBF) { @@ -1300,6 +2002,7 @@ extension Lexer.Cursor { fallthrough default: + scanningIndent = false if let peekedScalar = start.peekScalar(), peekedScalar.isValidIdentifierStartCodePoint { break } @@ -2194,7 +2897,7 @@ extension Lexer.Cursor { let rightBound = operEnd.isRightBound(isLeftBound: leftBound) // Match various reserved words. - if operEnd.input.baseAddress! - operStart.input.baseAddress! == 1 { + if operEnd.distance(from: operStart) == 1 { switch operStart.peek() { case "=": if leftBound != rightBound { @@ -2229,7 +2932,7 @@ extension Lexer.Cursor { default: break } - } else if operEnd.input.baseAddress! - operStart.input.baseAddress! == 2 { + } else if operEnd.distance(from: operStart) == 2 { switch (operStart.peek(), operStart.peek(at: 1)) { case ("-", ">"): // -> return (.arrow, error: nil) @@ -2298,7 +3001,7 @@ extension Lexer.Cursor { } } - if self.input.baseAddress! - tokStart.input.baseAddress! > 2 { + if self.distance(from: tokStart) > 2 { // If there is a "//" or "/*" in the middle of an identifier token, // it starts a comment. var ptr = tokStart diff --git a/Sources/SwiftParser/Lexer/LexemeSequence.swift b/Sources/SwiftParser/Lexer/LexemeSequence.swift index 1f9950aae18..e25fe0901b8 100644 --- a/Sources/SwiftParser/Lexer/LexemeSequence.swift +++ b/Sources/SwiftParser/Lexer/LexemeSequence.swift @@ -151,16 +151,23 @@ extension Lexer { @_spi(Testing) public static func tokenize( _ input: UnsafeBufferPointer, + mode: Parser.Mode = .swift, from startIndex: Int = 0, lookaheadTracker: UnsafeMutablePointer, experimentalFeatures: Parser.ExperimentalFeatures ) -> LexemeSequence { precondition(input.isEmpty || startIndex < input.endIndex) let startChar = startIndex == input.startIndex ? UInt8(ascii: "\0") : input[startIndex - 1] - let start = Cursor(input: input, previous: UInt8(ascii: "\0"), experimentalFeatures: experimentalFeatures) + let start = Cursor( + input: input, + previous: UInt8(ascii: "\0"), + mode: mode, + experimentalFeatures: experimentalFeatures + ) let cursor = Cursor( input: UnsafeBufferPointer(rebasing: input[startIndex...]), previous: startChar, + mode: mode, experimentalFeatures: experimentalFeatures ) return LexemeSequence( diff --git a/Sources/SwiftParser/ParseSourceFile.swift b/Sources/SwiftParser/ParseSourceFile.swift index e738ebdad34..1038aa62528 100644 --- a/Sources/SwiftParser/ParseSourceFile.swift +++ b/Sources/SwiftParser/ParseSourceFile.swift @@ -20,10 +20,12 @@ extension Parser { /// Parse the source code in the given string as Swift source file. See /// `Parser.init` for more details. public static func parse( - source: String + source: String, + mode: Parser.Mode = .swift ) -> SourceFileSyntax { return withParser( source: source, + mode: mode, maximumNestingLevel: nil, parseTransition: nil, swiftVersion: nil, @@ -35,11 +37,13 @@ extension Parser { @_spi(ExperimentalLanguageFeatures) public static func parse( source: UnsafeBufferPointer, + mode: Parser.Mode = .swift, swiftVersion: SwiftVersion? = nil, experimentalFeatures: ExperimentalFeatures ) -> SourceFileSyntax { return withParser( source: source, + mode: mode, maximumNestingLevel: nil, parseTransition: nil, swiftVersion: swiftVersion, @@ -51,11 +55,13 @@ extension Parser { /// `Parser.init` for more details. public static func parse( source: UnsafeBufferPointer, + mode: Parser.Mode = .swift, maximumNestingLevel: Int? = nil, swiftVersion: SwiftVersion? = nil ) -> SourceFileSyntax { return withParser( source: source, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: nil, swiftVersion: swiftVersion, @@ -87,9 +93,10 @@ extension Parser { @_disfavoredOverload public static func parseIncrementally( source: String, + mode: Parser.Mode = .swift, parseTransition: IncrementalParseTransition? ) -> (tree: SourceFileSyntax, lookaheadRanges: LookaheadRanges) { - let parseResult = parseIncrementally(source: source, parseTransition: parseTransition) + let parseResult = parseIncrementally(source: source, mode: mode, parseTransition: parseTransition) return (parseResult.tree, parseResult.lookaheadRanges) } @@ -101,11 +108,13 @@ extension Parser { @_disfavoredOverload public static func parseIncrementally( source: UnsafeBufferPointer, + mode: Parser.Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? ) -> (tree: SourceFileSyntax, lookaheadRanges: LookaheadRanges) { let parseResult = parseIncrementally( source: source, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition ) @@ -133,6 +142,7 @@ extension Parser { /// subsequent incremental parse public static func parseIncrementally( source: String, + mode: Parser.Mode = .swift, parseTransition: IncrementalParseTransition? ) -> IncrementalParseResult { // Drop the transition (forcing a full reparse) when the previous tree is @@ -140,6 +150,7 @@ extension Parser { let parseTransition = (parseTransition?.shouldCompact ?? false) ? nil : parseTransition return withParser( source: source, + mode: mode, maximumNestingLevel: nil, parseTransition: parseTransition, swiftVersion: nil, @@ -153,12 +164,14 @@ extension Parser { /// See doc comments in ``Parser/parseIncrementally(source:parseTransition:)-dj0z`` public static func parseIncrementally( source: UnsafeBufferPointer, + mode: Parser.Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? ) -> IncrementalParseResult { let parseTransition = (parseTransition?.shouldCompact ?? false) ? nil : parseTransition return withParser( source: source, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, swiftVersion: nil, diff --git a/Sources/SwiftParser/Parser.swift b/Sources/SwiftParser/Parser.swift index f5a90f6fc29..4d0d78ed653 100644 --- a/Sources/SwiftParser/Parser.swift +++ b/Sources/SwiftParser/Parser.swift @@ -91,6 +91,21 @@ /// tokens as needed to disambiguate a parse. However, because lookahead /// operates on a copy of the lexical stream, no input tokens are lost.. public struct Parser { + /// Identifies the mode in which the parser should run + public enum Mode: Equatable, Hashable { + /// Swift source file + case swift + + /// Literate Swift source file in Markdown + case markdown + + /// Literate Swift source file in reStructuredText + case reStructuredText + + /// Literate Swift source file in LaTeX + case laTeX + } + var arena: ParsingRawSyntaxArena /// A view of the sequence of lexemes in the input. @@ -210,6 +225,9 @@ public struct Parser { /// and can be freed after the initializer returns. When `false` the /// caller must keep `input` valid for the entire parse (see /// `withParser`). + /// - mode: The lexer mode; this can be `.swift` for a Swift file, or + /// `.markdown`, `.reStructuredText` or `.laTeX` for literate + /// Swift code. /// - maximumNestingLevel: To avoid overflowing the stack, the parser will /// stop if a nesting level greater than this value /// is reached. The nesting level is increased @@ -229,6 +247,7 @@ public struct Parser { /// - experimentalFeatures: The experimental features enabled for the parser. private init( buffer input: UnsafeBufferPointer, + mode: Mode = .swift, maximumNestingLevel: Int?, parseTransition: IncrementalParseTransition?, arena: ParsingRawSyntaxArena?, @@ -263,6 +282,7 @@ public struct Parser { self.lexemes = Lexer.tokenize( input, + mode: mode, lookaheadTracker: lookaheadTrackerOwner.lookaheadTracker, experimentalFeatures: experimentalFeatures ) @@ -277,6 +297,7 @@ public struct Parser { /// Private initializer for creating a ``Parser`` from the given string. private init( string input: String, + mode: Mode = .swift, maximumNestingLevel: Int?, parseTransition: IncrementalParseTransition?, swiftVersion: SwiftVersion?, @@ -287,6 +308,7 @@ public struct Parser { self = input.withUTF8 { buffer in Parser( buffer: buffer, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, arena: nil, @@ -303,6 +325,7 @@ public struct Parser { /// Initializes a ``Parser`` from the given string. public init( _ input: String, + mode: Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil, swiftVersion: SwiftVersion? = nil @@ -310,6 +333,7 @@ public struct Parser { // Chain to the private String initializer. self.init( string: input, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, swiftVersion: swiftVersion, @@ -323,6 +347,9 @@ public struct Parser { /// - input: An input buffer containing Swift source text. It is copied into /// a parser-owned buffer, so it can be freed after the initializer /// has been called. + /// - mode: The lexer mode; this can be `.swift` for a Swift file, or + /// `.markdown`, `.reStructuredText` or `.laTeX` for literate + /// Swift code. /// - maximumNestingLevel: To avoid overflowing the stack, the parser will /// stop if a nesting level greater than this value /// is reached. The nesting level is increased @@ -333,6 +360,7 @@ public struct Parser { /// parse, or `nil`. public init( _ input: UnsafeBufferPointer, + mode: Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil, swiftVersion: SwiftVersion? = nil @@ -341,6 +369,7 @@ public struct Parser { // free `input` after this initializer returns. self.init( buffer: input, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, arena: nil, @@ -355,6 +384,7 @@ public struct Parser { @_spi(ExperimentalLanguageFeatures) public init( _ input: String, + mode: Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil, swiftVersion: SwiftVersion? = nil, @@ -363,6 +393,7 @@ public struct Parser { // Chain to the private String initializer. self.init( string: input, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, swiftVersion: swiftVersion, @@ -375,6 +406,7 @@ public struct Parser { @_spi(ExperimentalLanguageFeatures) public init( _ input: UnsafeBufferPointer, + mode: Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil, arena: ParsingRawSyntaxArena? = nil, @@ -386,6 +418,7 @@ public struct Parser { // does not depend on `input` (its tokens are interned into `arena`). self.init( buffer: input, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, arena: arena, @@ -411,12 +444,14 @@ public struct Parser { @_spi(RawSyntax) public static func withParser( source input: UnsafeBufferPointer, + mode: Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil, body: (inout Parser) -> T ) -> T { return withParser( source: input, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, swiftVersion: nil, @@ -441,6 +476,7 @@ public struct Parser { @_spi(ExperimentalLanguageFeatures) public static func withParser( source input: UnsafeBufferPointer, + mode: Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil, swiftVersion: SwiftVersion?, @@ -449,6 +485,7 @@ public struct Parser { ) -> T { var parser = Parser( buffer: input, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, arena: nil, @@ -471,6 +508,7 @@ public struct Parser { @_spi(ExperimentalLanguageFeatures) public static func withParser( source input: String, + mode: Mode = .swift, maximumNestingLevel: Int? = nil, parseTransition: IncrementalParseTransition? = nil, swiftVersion: SwiftVersion?, @@ -482,6 +520,7 @@ public struct Parser { return input.withUTF8 { buffer in withParser( source: buffer, + mode: mode, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition, swiftVersion: swiftVersion, diff --git a/Tests/SwiftParserTest/Assertions.swift b/Tests/SwiftParserTest/Assertions.swift index 812f8384698..56ac05fafb7 100644 --- a/Tests/SwiftParserTest/Assertions.swift +++ b/Tests/SwiftParserTest/Assertions.swift @@ -572,6 +572,7 @@ extension ParserTestCase { func assertParse( _ markedSource: String, _ parse: @Sendable (inout Parser) -> some SyntaxProtocol = { SourceFileSyntax.parse(from: &$0) }, + mode: Parser.Mode = .swift, substructure expectedSubstructure: (some SyntaxProtocol)? = Optional.none, substructureAfterMarker: String = "START", diagnostics expectedDiagnostics: [DiagnosticSpec] = [], @@ -586,6 +587,7 @@ extension ParserTestCase { assertParse( markedSource, parse, + mode: mode, substructure: expectedSubstructure, substructureAfterMarker: substructureAfterMarker, diagnostics: expectedDiagnostics, @@ -627,6 +629,8 @@ extension ParserTestCase { /// to be used as locations in the following parameters. /// - parse: The function with which the source code should be parsed. /// Defaults to parsing as a source file. + /// - mode: The lexer mode (`.swift`, `.markdown`, `.reStructuredText`, + /// or `.laTeX`). /// - expectedSubstructure: Asserts the parsed syntax tree contains this structure. /// - substructureAfterMarker: Changes the position to start the structure /// assertion from, ie. allows matching a particular substructure rather @@ -643,6 +647,7 @@ extension ParserTestCase { func assertParse( _ markedSource: String, _ parse: @Sendable (inout Parser) -> some SyntaxProtocol = { SourceFileSyntax.parse(from: &$0) }, + mode: Parser.Mode = .swift, substructure expectedSubstructure: (some SyntaxProtocol)? = Optional.none, substructureAfterMarker: String = "START", diagnostics expectedDiagnostics: [DiagnosticSpec] = [], @@ -659,7 +664,7 @@ extension ParserTestCase { var (markerLocations, source) = extractMarkers(markedSource) markerLocations["START"] = 0 - var parser = Parser(source, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures) + var parser = Parser(source, mode: mode, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures) #if SWIFTPARSER_ENABLE_ALTERNATE_TOKEN_INTROSPECTION if !longTestsDisabled { parser.enableAlternativeTokenChoices() @@ -667,6 +672,37 @@ extension ParserTestCase { #endif let tree = parse(&parser) + let newSource = "\(tree)" + if newSource != source { + func hex(_ x: UInt8) -> String { + if x < 16 { + return "0\(String(x, radix: 16))" + } else { + return String(x, radix: 16) + } + } + func dumpHex(_ s: String) { + var tmp = s + tmp.withUTF8 { buffer in + for (ndx, byte) in buffer.enumerated() { + let terminator: String + if (ndx % 16) == 0 { + terminator = "\n" + } else { + terminator = " " + } + print(hex(byte), terminator: terminator) + } + print("") + } + } + print("Expected:") + dumpHex(source) + + print("Actual:") + dumpHex(newSource) + } + // Round-trip assertStringsEqualWithDiff( "\(tree)", @@ -748,6 +784,7 @@ extension ParserTestCase { if expectedDiagnostics.isEmpty && diags.isEmpty { assertBasicFormat( source: source, + mode: mode, parse: parse, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures, @@ -826,19 +863,21 @@ class TriviaRemover: SyntaxRewriter { func assertBasicFormat( source: String, + mode: Parser.Mode = .swift, parse: (inout Parser) -> some SyntaxProtocol, swiftVersion: Parser.SwiftVersion?, experimentalFeatures: Parser.ExperimentalFeatures, file: StaticString = #filePath, line: UInt = #line ) { - var parser = Parser(source, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures) + var parser = Parser(source, mode: mode, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures) let sourceTree = parse(&parser) let withoutTrivia = TriviaRemover(viewMode: .sourceAccurate).rewrite(sourceTree) let formatted = withoutTrivia.formatted() var formattedParser = Parser( formatted.description, + mode: .swift, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures ) diff --git a/Tests/SwiftParserTest/LiterateTests.swift b/Tests/SwiftParserTest/LiterateTests.swift new file mode 100644 index 00000000000..7dfbc25f194 --- /dev/null +++ b/Tests/SwiftParserTest/LiterateTests.swift @@ -0,0 +1,348 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftSyntax +import XCTest + +class LiterateTests: ParserTestCase { + func testLaTeX() { + assertParse( + #""" + \documentstyle{article} + \usepackage{listings} + \usepackage{color} + + \lstnewenvironment{swift}{\lstset{language=Swift,basicstyle=\small}}{} + + \title{Literate Swift in \LaTeX} + + \begin{document} + + We can write Swift code in a \LaTeX document using the \texttt{swift} + environment: + + \begin{swift} + // CHECK: LATEX + print("LATEX") + \end{swift} + + We can calculate $7!$ using the following code: + + \begin{swift} + // CHECK: 7! = 5040 + var result = 1 + for n in 2...7 { + result *= n + } + print("7! = \(result)") + \end{swift} + + \end{document} + """#, + mode: .laTeX + ) + } + + func testIndentedRST() { + assertParse( + #""" + Indented Code Blocks + ==================== + + In reStructuredText, code blocks are introduced with a double colon:: + + // CHECK: SIMPLE + print("SIMPLE") + + They run until the indentation level returns to the level of the paragraph that started them. + + So if we start a paragraph here:: + + // CHECK: INDENTED + // CHECK-NOT: AFTER-INDENTED + print("INDENTED") + + print("AFTER-INDENTED") + + is part of the text. + + We also support them in bulleted lists: + + * One + + * Two:: + + // CHECK: BULLET2 + print("BULLET2") + + * Three:: + + // CHECK: BULLET3 + print("BULLET3") + + * Four + + * Five:: + + // CHECK: BULLET5 + print("BULLET5") + + and in numbered lists: + + 1. One + + 2. Two:: + + // CHECK: NUMBERED2 + print("NUMBERED2") + + 3. Three:: + + // CHECK: NUMBERED3 + print("NUMBERED3") + + 4. Four + + 5. Five:: + + // CHECK: NUMBERED5 + print("NUMBERED5") + + 17. Seventeen:: + + // CHECK: NUMBERED17 + print("NUMBERED17") + + #. Automatic:: + + // CHECK: NUMBEREDAUTO + print("NUMBEREDAUTO") + + #. Auto2:: + + // CHECK: NUMBEREDAUTO2 + print("NUMBEREDAUTO2") + + Indented code blocks can contain indentation:: + + // CHECK: INDENTATION OK. SUM IS 55 + var sum = 0 + for n in 1...10 { + sum += n + } + print("INDENTATION OK. SUM IS \(sum)") + + They can also contain blank lines:: + + // CHECK: BLANK NO SPACES + + print("BLANK NO SPACES") + + with or without indentation:: + + // CHECK: BLANK SPACES + + print("BLANK SPACES") + """#, + mode: .reStructuredText + ) + } + + func testExplicitRST() { + assertParse( + #""" + Explicit Code Blocks + ==================== + + reStructuredText also supports explicit code blocks, using ``code-block`` and + ``sourcecode`` block markup. + + .. code-block:: + + // CHECK: SIMPLE + print("SIMPLE") + + .. sourcecode:: + + // CHECK: SOURCECODE + print("SOURCECODE") + + These can specify the language used; we only support ``swift``: + + .. code-block:: swift + + // CHECK: SWIFT + print("SWIFT") + + .. code-block:: pascal + + // CHECK-NOT: PASCAL + print("PASCAL") + + We also support a ``nocompile`` option for these, with or without the ``swift`` + language type: + + .. code-block:: + :nocompile: + + // CHECK-NOT: NOCOMPILE + print("NOCOMPILE") + + .. code-block:: swift + :nocompile: + + // CHECK-NOT: SWIFT-NOCOMPILE + print("SWIFT-NOCOMPILE") + + As with indented code blocks, they can contain indentation + + .. code-block:: swift + + // CHECK: INDENTATION OK. SUM IS 55 + var sum = 0 + for n in 1...10 { + sum += n + } + print("INDENTATION OK. SUM IS \(sum)") + + as well as blank lines + + .. code-block:: swift + + // CHECK: BLANK NO SPACES + + print("BLANK NO SPACES") + + with or without indentation + + .. code-block:: swift + + // CHECK: BLANK SPACES + + print("BLANK SPACES") + """#, + mode: .reStructuredText + ) + } + + func testIndentedMarkdown() { + assertParse( + #""" + Indented Code Block + =================== + + This should build a program that prints "OK": + + // CHECK: SIMPLE + print("SIMPLE") + + We also want to make sure that adding more blocks works: + + // CHECK: SECOND + print("SECOND") + + And that using more than four space works: + + // CHECK: FIVE SPACES + print("FIVE SPACES") + + Code blocks need a blank line before + // CHECK-NOT: NEED BLANK + print("NEED BLANK") + + Indented code blocks can contain indentation + + // CHECK: INDENTATION OK. SUM IS 55 + var sum = 0 + for n in 1...10 { + sum += n + } + print("INDENTATION OK. SUM IS \(sum)") + + Indented code blocks can also contain blank lines + + // CHECK: BLANK NO SPACES + + print("BLANK NO SPACES") + + with or without indentation + + // CHECK: BLANK SPACES + + print("BLANK SPACES") + """#, + mode: .markdown + ) + } + + func testFencedMarkdown() { + assertParse( + #""" + Fenced Code Blocks + ================== + + A simple fenced code block: + + ``` + // CHECK: SIMPLE + print("SIMPLE") + ``` + + Fenced code blocks can use tildes instead of backticks: + + ~~~ + // CHECK: TILDE + print("TILDE") + ~~~ + + This is not a valid code block: + + ``` print("NOT VALID") // CHECK-NOT: NOT_VALID ``` + + This *is* a valid code block: + + ~~~ swift ``` + // CHECK: BACKTICKS IN INFO + print("BACKTICKS IN INFO") + ~~~ + + Fenced code blocks do *not* need a blank line before: + ``` + // CHECK: NO BLANK REQUIRED + print("NO BLANK REQUIRED") + ``` + + We ignore fenced blocks for other languages: + + ```pascal + // CHECK-NOT: IGNORED OTHER LANGUAGE + print("IGNORED OTHER LANGUAGE") + ``` + + We process *swift* blocks + + ```swift + // CHECK: SWIFT BLOCK + print("SWIFT BLOCK") + ``` + + unless they use the `nocompile` keyword: + + ```swift nocompile + // CHECK-NOT: SWIFT NOCOMPILE + print("SWIFT NOCOMPILE") + ``` + """#, + mode: .markdown + ) + } +}