blob literal

Author: wickwirew

Sources/Compiler/Parse/Lexer.swift

@@ -82,7 +82,9 @@ struct Lexer {
 
         let peek = peek
 
-        if current.isLetter {
+        // The x' is actually the start of a blob literal which
+        // will be handled later in the switch
+        if current.isLetter, !(current == "x" && peek == "'") {
             return parseWord()
         }
 
@@ -140,7 +142,14 @@ struct Lexer {
         case ("|", _): return consumeSingle(of: .pipe)
         case ("^", _): return consumeSingle(of: .carrot)
         case ("~", _): return consumeSingle(of: .tilde)
-        case ("'", _): return parseString()
+        case ("'", _):
+            let (contents, location) = parseStringContents()
+            return Token(kind: .string(contents), location: location)
+        case ("x", "'"):
+            let start = startLocation()
+            advance() // past the x
+            let (contents, _) = parseStringContents()
+            return Token(kind: .blob(contents), location: location(from: start))
         case ("\"", _): return parseEscapedIdentifier(closing: "\"")
         case ("[", _): return parseEscapedIdentifier(closing: "]")
         case ("`", _): return parseEscapedIdentifier(closing: "`")
@@ -331,7 +340,9 @@ struct Lexer {
         return Token(kind: .hex(value), location: location)
     }
 
-    private mutating func parseString() -> Token {
+    /// Gets the raw string between two `'`. Assumes that the starting char
+    /// is a quote and will consume both the starting and end.
+    private mutating func parseStringContents() -> (Substring, SourceLocation) {
         let tokenStart = startLocation()
         advance()
         let stringStart = currentIndex
@@ -348,7 +359,7 @@ struct Lexer {
             diagnostics.add(.init("Unterminated string", at: location(from: tokenStart)))
         }
 
-        return Token(kind: .string(source[stringRange]), location: location(from: tokenStart))
+        return (source[stringRange], location(from: tokenStart))
     }
 
     private mutating func skipWhitespace() {

Sources/Compiler/Parse/Parsers.swift

@@ -2058,7 +2058,7 @@ enum Parsers {
         precedence: Operator.Precedence
     ) throws -> any ExprSyntax {
         switch state.current.kind {
-        case .double, .string, .int, .hex, .currentDate, .currentTime, .currentTimestamp, .true, .false:
+        case .double, .string, .int, .hex, .currentDate, .currentTime, .currentTimestamp, .true, .false, .blob:
             return literal(state: &state)
         case .identifier, .star:
             return try columnExpr(state: &state, schema: nil, table: nil)
@@ -2528,6 +2528,8 @@ enum Parsers {
             kind = .numeric(Double(value), isInt: true)
         case let .string(value):
             kind = .string(value)
+        case let .blob(value):
+            kind = .blob(value)
         case .true:
             kind = .true
         case .false:

Sources/Compiler/Parse/Token.swift

@@ -176,6 +176,7 @@ struct Token {
         case double(Double)
         case int(Int)
         case hex(Int)
+        case blob(Substring)
 
         case abort
         case action
@@ -387,6 +388,7 @@ struct Token {
             case let .double(value): value.description
             case let .int(value): value.description
             case let .hex(value): value.description
+            case let .blob(value): value.description
             case .abort: "ABORT"
             case .action: "ACTION"
             case .add: "ADD"

Tests/CompilerTests/LexerTests.swift

@@ -31,6 +31,11 @@ class LexerTests: XCTestCase {
         XCTAssertEqual(tokens, [.int(100), .double(20.2), .int(123), .double(123.4), .double(1e2), .double(3.2e-3), .hex(0xFF), .eof])
     }
 
+    func testBlob() throws {
+        let tokens = tokens(of: "x'some words' x'select'")
+        XCTAssertEqual(tokens, [.blob("some words"), .blob("select"), .eof])
+    }
+    
     func testOperators() throws {
         let tokens = tokens(of: "<< <= >> >= || == != <> -> ->> * . ( ) , + - / % < > & | ^ ~")