Tokens now always track their position in the source command and aways use this as the source for ToString.

Author: RyanLamansky

SqlServerSimulator/Parser/Token.cs

@@ -5,10 +5,26 @@
 /// </summary>
 abstract class Token
 {
-    private protected Token()
+    /// <summary>
+    /// The original SQL command that contains this token.
+    /// </summary>
+    private readonly string command;
+
+    private readonly int index, length;
+
+    private protected Token(string command, int index, int length)
     {
+        System.Diagnostics.Debug.Assert(index + length <= command.Length);
+        this.command = command;
+        this.index = index;
+        this.length = length;
     }
 
+    /// <summary>
+    /// Returns a span containing the portion of the original command this token is based upon.
+    /// </summary>
+    public ReadOnlySpan<char> Source => command.AsSpan(index, length);
+
     // This is used for various error messages even though tokens are not directly accessible to user code.
-    public abstract override string ToString();
+    public sealed override string ToString() => command.Substring(index, length);
 }

SqlServerSimulator/Parser/Tokenizer.cs

@@ -1,5 +1,4 @@
 using SqlServerSimulator.Parser.Tokens;
-using System.Text;
 
 namespace SqlServerSimulator.Parser;
 
@@ -23,13 +22,13 @@ static class Tokenizer
             '@' => ParseAtOrDoubleAtPrefixedString(command, ref index),
             '-' => ParseMinusOrComment(command, ref index),
             '[' => ParseBracketDelimitedString(command, ref index),
-            '+' => new Plus(),
-            '*' => new Asterisk(),
-            '(' => new OpenParentheses(),
-            ')' => new CloseParentheses(),
-            ',' => new Comma(),
-            '.' => new Period(),
-            ';' => new StatementTerminator(),
+            '+' => new Plus(command, index),
+            '*' => new Asterisk(command, index),
+            '(' => new OpenParentheses(command, index),
+            ')' => new CloseParentheses(command, index),
+            ',' => new Comma(command, index),
+            '.' => new Period(command, index),
+            ';' => new StatementTerminator(command, index),
             _ => throw new NotSupportedException($"Simulated tokenizer doesn't know what to do with character '{command[index]}' at index {index}.")
         };
 
@@ -80,7 +79,7 @@ private static Numeric ParseNumeric(string command, ref int index)
             break;
         }
 
-        return new(new StringBuilder(command, start, index - start, index-- - start));
+        return new(command, start, index-- - start);
     }
 
     private static Token ParseAtOrDoubleAtPrefixedString(string command, ref int index)
@@ -110,19 +109,20 @@ private static Token ParseAtOrDoubleAtPrefixedString(string command, ref int ind
             break;
         }
 
-        return doubleAt
-            ? new DoubleAtPrefixedString(new StringBuilder(command, start + 2, index-- - (start + 2), index - (start + 2) - 1))
-            : new AtPrefixedString(new StringBuilder(command, start + 1, index-- - (start + 1), index - (start + 1) - 1));
+        return doubleAt ?
+            new DoubleAtPrefixedString(command, start, index-- - start) :
+            new AtPrefixedString(command, start, index-- - start);
     }
 
     private static Token ParseMinusOrComment(string command, ref int index)
     {
+        var start = index;
         if (++index == command.Length)
-            return new Minus();
+            return new Minus(command, --index);
         if (command[index] != '-')
         {
             index--;
-            return new Minus();
+            return new Minus(command, index);
         }
 
         while (++index < command.Length)
@@ -131,11 +131,11 @@ private static Token ParseMinusOrComment(string command, ref int index)
             {
                 case '\r':
                 case '\n':
-                    return new Comment();
+                    return new Comment(command, start, --index);
             }
         }
 
-        return new Comment();
+        return new Comment(command, start, --index);
     }
 
     private static BracketDelimitedString ParseBracketDelimitedString(string command, ref int index)
@@ -150,6 +150,6 @@ private static BracketDelimitedString ParseBracketDelimitedString(string command
             break;
         }
 
-        return new(new StringBuilder(command, start + 1, index - start - 2, index-- - start - 2));
+        return new(command.Substring(start + 1, index - start - 2), command, start, index-- - start);
     }
 }

SqlServerSimulator/Parser/Tokens/Asterisk.cs

@@ -1,6 +1,5 @@
 namespace SqlServerSimulator.Parser.Tokens;
 
-sealed class Asterisk : Token
+sealed class Asterisk(string command, int index) : Token(command, index, 1)
 {
-    public override string ToString() => "*";
 }

SqlServerSimulator/Parser/Tokens/AtPrefixedString.cs

@@ -1,8 +1,7 @@
-using System.Text;
-
+
 namespace SqlServerSimulator.Parser.Tokens;
 
-sealed class AtPrefixedString(StringBuilder buffer) : StringToken(buffer)
+sealed class AtPrefixedString(string command, int index, int length) : StringToken(command, index, length)
 {
-    public override string ToString() => $"@{Value}";
+    public override ReadOnlySpan<char> Span => Source[1..];
 }

SqlServerSimulator/Parser/Tokens/BracketDelimitedString.cs

@@ -1,8 +1,8 @@
-using System.Text;
+namespace SqlServerSimulator.Parser.Tokens;
 
-namespace SqlServerSimulator.Parser.Tokens;
-
-sealed class BracketDelimitedString(StringBuilder buffer) : Name(buffer)
+sealed class BracketDelimitedString(string value, string command, int index, int length) : Name(command, index, length)
 {
-    public override string ToString() => $"[{Value}]";
+    public override ReadOnlySpan<char> Span => Value.AsSpan();
+
+    public override string Value { get; } = value;
 }

SqlServerSimulator/Parser/Tokens/CloseParentheses.cs

@@ -1,6 +1,5 @@
 namespace SqlServerSimulator.Parser.Tokens;
 
-sealed class CloseParentheses : Token
+sealed class CloseParentheses(string command, int index) : Token(command, index, 1)
 {
-    public override string ToString() => ")";
 }

SqlServerSimulator/Parser/Tokens/Comma.cs

@@ -1,6 +1,5 @@
 namespace SqlServerSimulator.Parser.Tokens;
 
-sealed class Comma : Token
+sealed class Comma(string command, int index) : Token(command, index, 1)
 {
-    public override string ToString() => ",";
 }

SqlServerSimulator/Parser/Tokens/Comment.cs

@@ -1,6 +1,5 @@
 namespace SqlServerSimulator.Parser.Tokens;
 
-sealed class Comment : Token
+sealed class Comment(string command, int index, int length) : Token(command, index, length)
 {
-    public override string ToString() => "/* Comment */";
 }

SqlServerSimulator/Parser/Tokens/DoubleAtPrefixedString.cs

@@ -1,13 +1,11 @@
-using System.Text;
+namespace SqlServerSimulator.Parser.Tokens;
 
-namespace SqlServerSimulator.Parser.Tokens;
-
-sealed class DoubleAtPrefixedString(StringBuilder buffer) : StringToken(buffer)
+sealed class DoubleAtPrefixedString(string command, int index, int length) : StringToken(command, index, length)
 {
-    public AtAtKeyword Parse() => !Enum.TryParse<AtAtKeyword>(Value, true, out var result)
-        ? throw new NotSupportedException($"Simulated command processor doesn't know what to do with `{Value}`.")
-        : result;
+    public override ReadOnlySpan<char> Span => Source[2..];
 
-    public override string ToString() => $"@@{Value}";
+    public AtAtKeyword Parse() => !Enum.TryParse<AtAtKeyword>(Span, true, out var result)
+        ? throw new NotSupportedException($"Simulated command processor doesn't know what to do with `{Span}`.")
+        : result;
 }
 

SqlServerSimulator/Parser/Tokens/Minus.cs

@@ -1,6 +1,5 @@
 namespace SqlServerSimulator.Parser.Tokens;
 
-sealed class Minus : Token
+sealed class Minus(string command, int index) : Token(command, index, 1)
 {
-    public override string ToString() => "-";
 }