Tokenizer position and advancement is now directly handled by the parser context.

Author: RyanLamansky

SqlServerSimulator/Parser/ParserContext.cs

@@ -7,57 +7,36 @@ namespace SqlServerSimulator.Parser;
 /// <summary>
 /// Organizes relevant information for parsing of SQL commands.
 /// </summary>
-internal sealed class ParserContext : IDisposable
+internal sealed class ParserContext(SimulatedDbCommand command)
 {
-    public readonly SimulatedDbCommand Command;
-#if DEBUG
-    private readonly TokenArrayEnumerator tokens;
-#else
-    private readonly IEnumerator<Token> tokens;
-#endif
-    public Token? Token { get; private set; }
-    private readonly FrozenDictionary<string, (string Name, (DataType type, object? Value) TypeValue)> variables;
+#pragma warning disable CA2213 // Disposable fields should be disposed
+    public readonly SimulatedDbCommand Command = command;
+#pragma warning restore CA2213 // Suppressed because ParserContext doesn't own the command object.
 
-    public ParserContext(SimulatedDbCommand command)
-    {
-        this.Command = command;
-
-        this.variables = command
-            .Parameters
-            .Cast<DbParameter>()
-            .Select(parameter =>
-            {
-                var name = parameter.ParameterName;
-                var type = DataType.GetByDbType(parameter.DbType);
-                return (Name: name.StartsWith('@') ? name[1..] : name, TypeValue: (DataType: type, Value: parameter.Value is null ? null : type.ConvertFrom(parameter.Value)));
-            })
-            .ToFrozenDictionary(tuple => tuple.Name, StringComparer.InvariantCultureIgnoreCase);
+    private readonly string commandText = string.IsNullOrEmpty(command.CommandText) ?
+        throw new InvalidOperationException("ExecuteReader: CommandText property has not been initialized") :
+        command.CommandText;
 
-#if DEBUG
-        this.tokens = new TokenArrayEnumerator(command.CommandText);
-#else
-        this.tokens = TokenizerIterator(command.CommandText).GetEnumerator();
-#endif
-    }
+    /// <summary>
+    /// The tokenizer position within <see cref="commandText"/>.
+    /// </summary>
+    private int index = -1;
 
     /// <summary>
-    /// Converts the redesigned tokenizer to a traditional enumerable for compatibility with the existing logic.
-    /// This should be phased out eventually.
+    /// The most recently identified token in the command string.
     /// </summary>
-    static IEnumerable<Token> TokenizerIterator(string? commandText)
-    {
-        if (string.IsNullOrEmpty(commandText))
-            throw new InvalidOperationException("ExecuteReader: CommandText property has not been initialized");
+    public Token? Token { get; private set; }
 
-        var index = -1;
-        while (Tokenizer.NextToken(commandText, ref index) is Token token)
+    private readonly FrozenDictionary<string, (string Name, (DataType type, object? Value) TypeValue)> variables = command
+        .Parameters
+        .Cast<DbParameter>()
+        .Select(parameter =>
         {
-            if (token is Whitespace)
-                continue;
-
-            yield return token;
-        }
-    }
+            var name = parameter.ParameterName;
+            var type = DataType.GetByDbType(parameter.DbType);
+            return (Name: name.StartsWith('@') ? name[1..] : name, TypeValue: (DataType: type, Value: parameter.Value is null ? null : type.ConvertFrom(parameter.Value)));
+        })
+        .ToFrozenDictionary(tuple => tuple.Name, StringComparer.InvariantCultureIgnoreCase);
 
     public Simulation Simulation => Command.simulation;
 
@@ -77,8 +56,7 @@ static IEnumerable<Token> TokenizerIterator(string? commandText)
     /// </summary>
     public void MoveNextOptional()
     {
-        var enumerator = this.tokens;
-        this.Token = enumerator.MoveNext() ? enumerator.Current : null;
+        _ = MoveNext();
     }
 
     /// <summary>
@@ -88,8 +66,7 @@ public void MoveNextOptional()
     /// <returns>The next token if the enumerator was advanced, otherwise null.</returns>
     public Token? GetNextOptional()
     {
-        var enumerator = this.tokens;
-        return this.Token = enumerator.MoveNext() ? enumerator.Current : null;
+        return MoveNext() ? this.Token : null;
     }
 
     /// <summary>
@@ -100,9 +77,8 @@ public void MoveNextOptional()
     /// <exception cref="SimulatedSqlException">Incorrect syntax near '{token}'.</exception>
     public Token GetNextRequired()
     {
-        var enumerator = this.tokens;
-        var previous = enumerator.Current;
-        return enumerator.MoveNext() ? this.Token = enumerator.Current : throw SimulatedSqlException.SyntaxErrorNear(previous);
+        var previous = this.Token;
+        return MoveNext() ? this.Token! : throw SimulatedSqlException.SyntaxErrorNear(previous);
     }
 
     /// <summary>
@@ -115,16 +91,9 @@ public Token GetNextRequired()
     public T GetNextRequired<T>()
         where T : Token
     {
-        var enumerator = this.tokens;
-        var previous = enumerator.Current;
+        var previous = this.Token;
 
-        if (enumerator.MoveNext() && enumerator.Current is T current)
-        {
-            this.Token = current;
-            return current;
-        }
-
-        throw SimulatedSqlException.SyntaxErrorNear(previous);
+        return MoveNext() && this.Token is T current ? current : throw SimulatedSqlException.SyntaxErrorNear(previous);
     }
 
     /// <summary>
@@ -135,10 +104,10 @@ public T GetNextRequired<T>()
     public void MoveNextRequired<T>()
         where T : Token
     {
-        var enumerator = this.tokens;
-        var previous = enumerator.Current;
+        var previous = this.Token;
 
-        this.Token = enumerator.MoveNext() && enumerator.Current is T current ? (Token)current : throw SimulatedSqlException.SyntaxErrorNear(previous);
+        if (!MoveNext() || this.Token is not T)
+            throw SimulatedSqlException.SyntaxErrorNear(previous);
     }
 
     /// <summary>
@@ -147,98 +116,55 @@ public void MoveNextRequired<T>()
     /// <exception cref="SimulatedSqlException">Incorrect syntax near '{token}'.</exception>
     public void MoveNextRequired()
     {
-        var enumerator = this.tokens;
-        var previous = enumerator.Current;
-        this.Token = enumerator.MoveNext() ? enumerator.Current : throw SimulatedSqlException.SyntaxErrorNear(previous);
+        var previous = this.Token;
+        if (!MoveNext())
+            throw SimulatedSqlException.SyntaxErrorNear(previous);
     }
 
-    private bool isDisposed;
-
-    private void Dispose(bool disposing)
+    private bool MoveNext()
     {
-        if (isDisposed)
-            return;
-
-        if (disposing)
+        while (Tokenizer.NextToken(commandText, ref index) is Token token)
         {
-            this.tokens.Dispose();
-            this.Token = null;
+            if (token is Whitespace)
+                continue;
+
+            this.Token = token;
+            return true;
         }
 
-        isDisposed = true;
-    }
-
-    public void Dispose()
-    {
-        // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
-        Dispose(true);
-        GC.SuppressFinalize(this);
+        this.Token = null;
+        return false;
     }
 
 #if DEBUG
-    public override string? ToString() => this.tokens.ToString();
-
     /// <summary>
-    /// Simplifies parser debugging by providing a useful string representation of the token enumeration.
-    /// Specifically, when calling <see cref="ToString"/> tokens are separated by '·' and the <see cref="Current"/> token is wrapped by '»' and '«'.
+    /// Returns a string representation of the tokenized command.
+    /// The <see cref="Token"/> token is wrapped by '»' and '«'.
     /// </summary>
-    /// <param name="command">The SQL command to process.</param>
-    /// <remarks>This should only be included in debug builds because it reduces parsing efficiency.</remarks>
-    private sealed class TokenArrayEnumerator(string? command) : IEnumerator<Token>
+    /// <returns>The string representation.</returns>
+    public override string ToString()
     {
-        /// <summary>
-        /// Retains the full results of tokenization.
-        /// This is less efficient than streaming the results, but enables this class's debugging-friendly <see cref="ToString"/>.
-        /// </summary>
-        private readonly Token[] source = [.. TokenizerIterator(command)];
-
-        public int Index { get; private set; } = -1;
-
-        public Token Current => source[Index];
-
-        object System.Collections.IEnumerator.Current => Current;
-
-        public void Dispose()
+        var command = this.commandText;
+        Span<char> result = stackalloc char[command.Length + 2];
+        if (index < 0)
         {
+            result[0] = '»';
+            result[1] = '«';
+            command.CopyTo(result[2..]);
         }
-
-        public bool MoveNext()
+        else if (index >= command.Length)
         {
-            var newIndex = checked(Index + 1);
-            if (newIndex >= source.Length)
-                return false;
-
-            Index = newIndex;
-            return true;
+            command.CopyTo(result);
+            result[^2] = '»';
+            result[^1] = '«';
         }
-
-        public void Reset() => Index = -1;
-
-        /// <summary>
-        /// Returns a string representation of the tokenized command.
-        /// Tokens are separated by '·' and the <see cref="Current"/> token is wrapped by '»' and '«'.
-        /// </summary>
-        /// <returns>The string representation.</returns>
-        public override string ToString()
+        else
         {
-            var result = new System.Text.StringBuilder();
-            var source = this.source;
-
-            for (var i = 0; i < source.Length; i++)
-            {
-                var token = source[i];
-                if (i != Index)
-                {
-                    _ = result.Append(token).Append('·');
-                    continue;
-                }
-
-                _ = result.Append('»').Append(token).Append('«').Append('·');
-            }
-
-            return result.ToString(0, result.Length - 1);
-
+            System.Diagnostics.Debug.Assert(this.Token is not null);
+            this.Token.Highlight(result);
         }
+
+        return new string(result);
     }
 #endif
 }

SqlServerSimulator/Parser/Token.cs

@@ -27,4 +27,20 @@ private protected Token(string command, int index, int length)
 
     // This is used for various error messages even though tokens are not directly accessible to user code.
     public sealed override string ToString() => command.Substring(index, length);
+
+#if DEBUG
+    /// <summary>
+    /// Identifies this token within the scope of the full command by wrapping it with '»' and '«';
+    /// </summary>
+    public void Highlight(Span<char> result)
+    {
+        var command = this.command.AsSpan();
+
+        command[..this.index].CopyTo(result);
+        result[index] = '»';
+        this.Source.CopyTo(result[(index + 1)..]);
+        result[index + 1 + this.length] = '«';
+        command[(index + length)..].CopyTo(result[(index + length + 2)..]);
+    }
+#endif
 }

SqlServerSimulator/Simulation.cs

@@ -30,7 +30,7 @@ public Simulation()
 
     internal IEnumerable<SimulatedStatementOutcome> CreateResultSetsForCommand(SimulatedDbCommand command)
     {
-        using var context = new ParserContext(command);
+        var context = new ParserContext(command);
 
         while (context.GetNextOptional() is not null)
         {