fix: resolve 5 of 6 known engine limitations - IS NULL/IS NOT NULL across all WHERE paths with DBNull, German collation IgnoreNonSpace, COALESCE/scalar func parsing, trailing token error recovery, LINQ enum Convert. Dispose improved but deadlock re-skipped. 1490 tests pass.

MPCoreDeveloper · MPCoreDeveloper · commit a0d01679cea2 · 2026-03-14T11:01:55.000+01:00
diff --git a/src/SharpCoreDB/CultureInfoCollation.cs b/src/SharpCoreDB/CultureInfoCollation.cs
@@ -114,7 +114,11 @@ public int Compare(string? left, string? right, string localeName, bool ignoreCa
         if (right is null) return 1;
 
         var compareInfo = GetCompareInfo(localeName);
-        var options = ignoreCase ? CompareOptions.IgnoreCase : CompareOptions.None;
+        // IgnoreNonSpace gives primary-level (base letter) comparison:
+        // ß = ss in German, accented chars equivalent (é = e, etc.)
+        var options = ignoreCase
+            ? CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace
+            : CompareOptions.None;
         return compareInfo.Compare(left, right, options);
     }
 
@@ -133,7 +137,11 @@ public bool Equals(string? left, string? right, string localeName, bool ignoreCa
         if (left is null || right is null) return false;
 
         var compareInfo = GetCompareInfo(localeName);
-        var options = ignoreCase ? CompareOptions.IgnoreCase : CompareOptions.None;
+        // IgnoreNonSpace gives primary-level (base letter) comparison:
+        // ß = ss in German, accented chars equivalent (é = e, etc.)
+        var options = ignoreCase
+            ? CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace
+            : CompareOptions.None;
         return compareInfo.Compare(left, right, options) == 0;
     }
 
diff --git a/src/SharpCoreDB/DataStructures/Table.Scanning.cs b/src/SharpCoreDB/DataStructures/Table.Scanning.cs
@@ -144,15 +144,16 @@ private int FindPatternInRowData(ReadOnlySpan<byte> rowData, byte pattern)
 
     /// <summary>
     /// Evaluates a WHERE clause against a row.
-    /// Supports operators: equals, not equals, greater than, less than, greater or equal, less or equal
+    /// Supports operators: equals, not equals, greater than, less than, greater or equal, less or equal,
+    /// IS NULL, IS NOT NULL.
     /// </summary>
     private bool EvaluateWhere(Dictionary<string, object> row, string? where)
     {
         if (string.IsNullOrEmpty(where)) return true;
-        
+
         var parts = where.Split(' ', StringSplitOptions.RemoveEmptyEntries);
         if (parts.Length < 3) return true;
-        
+
         var columnName = parts[0].Trim('"', '[', ']', '`');
 
         // ✅ FIX: Strip table alias prefix from column names (e.g., "u"."Username" → Username)
@@ -164,9 +165,26 @@ private bool EvaluateWhere(Dictionary<string, object> row, string? where)
         {
             columnName = columnName[(dotIdx + 1)..].Trim('"', '[', ']', '`');
         }
+
+        // Handle IS NOT NULL (4 tokens: column IS NOT NULL)
+        if (parts.Length >= 4
+            && parts[1].Equals("IS", StringComparison.OrdinalIgnoreCase)
+            && parts[2].Equals("NOT", StringComparison.OrdinalIgnoreCase)
+            && parts[3].Equals("NULL", StringComparison.OrdinalIgnoreCase))
+        {
+            return row.TryGetValue(columnName, out var v) && v is not null && v is not DBNull;
+        }
+
+        // Handle IS NULL (3 tokens: column IS NULL)
+        if (parts[1].Equals("IS", StringComparison.OrdinalIgnoreCase)
+            && parts[2].Equals("NULL", StringComparison.OrdinalIgnoreCase))
+        {
+            return !row.TryGetValue(columnName, out var v) || v is null || v is DBNull;
+        }
+
         var op = parts[1];
         var value = parts[2].Trim('"').Trim((char)39);
-        
+
         if (!row.TryGetValue(columnName, out var rowValue) || rowValue == null)
             return false;
 
diff --git a/src/SharpCoreDB/Linq/GenericLinqToSqlTranslator.Core.cs b/src/SharpCoreDB/Linq/GenericLinqToSqlTranslator.Core.cs
@@ -78,6 +78,8 @@ private void Visit(Expression? expression)
                 break;
 
             case ExpressionType.Not:
+            case ExpressionType.Convert:
+            case ExpressionType.ConvertChecked:
                 VisitUnary((UnaryExpression)expression);
                 break;
 
diff --git a/src/SharpCoreDB/Linq/GenericLinqToSqlTranslator.Expressions.cs b/src/SharpCoreDB/Linq/GenericLinqToSqlTranslator.Expressions.cs
@@ -80,6 +80,24 @@ private void VisitUnary(UnaryExpression expression)
             Visit(expression.Operand);
             _sql.Append(')');
         }
+        else if (expression.NodeType is ExpressionType.Convert or ExpressionType.ConvertChecked)
+        {
+            // Unwrap enum/numeric casts — visit the inner operand directly
+            if (expression.Operand is ConstantExpression constant)
+            {
+                // Resolve enum values to their underlying integral value
+                var value = constant.Value;
+                if (value is not null && value.GetType().IsEnum)
+                {
+                    value = System.Convert.ChangeType(value, Enum.GetUnderlyingType(value.GetType()));
+                }
+                VisitConstant(System.Linq.Expressions.Expression.Constant(value, expression.Type));
+            }
+            else
+            {
+                Visit(expression.Operand);
+            }
+        }
         else
         {
             Visit(expression.Operand);
diff --git a/src/SharpCoreDB/Services/EnhancedSqlParser.Expressions.cs b/src/SharpCoreDB/Services/EnhancedSqlParser.Expressions.cs
@@ -63,6 +63,22 @@ private ExpressionNode ParseComparisonExpression()
     {
         var left = ParsePrimaryExpression();
 
+        // Check for IS NULL / IS NOT NULL
+        if (MatchKeyword("IS"))
+        {
+            bool isNot = MatchKeyword("NOT");
+            if (MatchKeyword("NULL"))
+            {
+                return new BinaryExpressionNode
+                {
+                    Position = _position,
+                    Left = left,
+                    Operator = isNot ? "IS NOT NULL" : "IS NULL",
+                    Right = new LiteralNode { Position = _position, Value = null }
+                };
+            }
+        }
+
         // Check for IN expression
         if (MatchKeyword("IN"))
         {
diff --git a/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs b/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs
@@ -179,6 +179,38 @@ private List<ColumnNode> ParseSelectColumns()
             }
 
             // Parse table.column or column
+            // First check for parenthesized expression (scalar subquery or grouped expression)
+            var remaining = _sql.Substring(_position);
+            if (remaining.TrimStart().StartsWith('('))
+            {
+                var expr = ParsePrimaryExpression();
+                column.Expression = expr;
+                column.Name = expr is SubqueryExpressionNode ? "(subquery)" : "(expr)";
+
+                if (MatchKeyword("AS"))
+                    column.Alias = ConsumeIdentifier();
+
+                return column;
+            }
+
+            // Check for scalar function calls (COALESCE, IIF, NULLIF, etc.)
+            var scalarFuncMatch = Regex.Match(
+                remaining,
+                @"^\s*(\w+)\s*\(",
+                RegexOptions.IgnoreCase);
+            if (scalarFuncMatch.Success)
+            {
+                // This is a scalar function — parse as expression
+                var funcExpr = ParseFunctionCall();
+                column.Expression = funcExpr;
+                column.Name = funcExpr.FunctionName;
+
+                if (MatchKeyword("AS"))
+                    column.Alias = ConsumeIdentifier();
+
+                return column;
+            }
+
             var identifier = ConsumeIdentifier();
             if (identifier is null)
                 return null;
diff --git a/src/SharpCoreDB/Services/EnhancedSqlParser.cs b/src/SharpCoreDB/Services/EnhancedSqlParser.cs
@@ -50,7 +50,7 @@ public partial class EnhancedSqlParser(ISqlDialect? dialect = null)
         {
             var keyword = PeekKeyword();
 
-            return keyword?.ToUpperInvariant() switch
+            var result = keyword?.ToUpperInvariant() switch
             {
                 "SELECT" => ParseSelect(),
                 "INSERT" => ParseInsert(),
@@ -60,6 +60,15 @@ public partial class EnhancedSqlParser(ISqlDialect? dialect = null)
                 "ALTER" => ParseAlter(),
                 _ => throw new InvalidOperationException($"Unsupported statement type: {keyword}")
             };
+
+            // Validate no unparsed trailing tokens remain
+            var remaining = _sql.Substring(_position).Trim();
+            if (remaining.Length > 0)
+            {
+                RecordError($"Unexpected trailing content: '{remaining}'");
+            }
+
+            return result;
         }
         catch (Exception ex)
         {
diff --git a/src/SharpCoreDB/Services/SqlAst.Nodes.cs b/src/SharpCoreDB/Services/SqlAst.Nodes.cs
@@ -108,6 +108,12 @@ public class ColumnNode : SqlNode
     /// </summary>
     public double? AggregateArgument { get; set; }
 
+    /// <summary>
+    /// Gets or sets a parsed expression for scalar functions (e.g., COALESCE, IIF, NULLIF).
+    /// When set, this expression represents the full column value computation.
+    /// </summary>
+    public ExpressionNode? Expression { get; set; }
+
     /// <inheritdoc/>
     public override TResult Accept<TResult>(ISqlVisitor<TResult> visitor) => visitor.VisitColumn(this);
 }
diff --git a/src/SharpCoreDB/Services/SqlParser.Helpers.cs b/src/SharpCoreDB/Services/SqlParser.Helpers.cs
@@ -263,6 +263,24 @@ private static bool EvaluateSimpleCondition(Dictionary<string, object> row, stri
     {
         var key = parts[0].Trim();
         var op = parts[1].Trim();
+
+        // Handle IS NOT NULL (parts: key, IS, NOT, NULL)
+        if (op.Equals("IS", StringComparison.OrdinalIgnoreCase)
+            && parts.Length >= 4
+            && parts[2].Trim().Equals("NOT", StringComparison.OrdinalIgnoreCase)
+            && parts[3].Trim().Equals("NULL", StringComparison.OrdinalIgnoreCase))
+        {
+            return row.TryGetValue(key, out var v) && v is not null && v is not DBNull;
+        }
+
+        // Handle IS NULL (parts: key, IS, NULL)
+        if (op.Equals("IS", StringComparison.OrdinalIgnoreCase)
+            && parts.Length >= 3
+            && parts[2].Trim().Equals("NULL", StringComparison.OrdinalIgnoreCase))
+        {
+            return !row.TryGetValue(key, out var v) || v is null || v is DBNull;
+        }
+
         var value = parts[2].Trim().Trim('\'');
 
         if (value.Equals("NULL", StringComparison.OrdinalIgnoreCase))
@@ -292,18 +310,42 @@ private static bool EvaluateComplexCondition(Dictionary<string, object> row, str
         {
             var key = parts[i].Trim();
             var op = parts[i + 1].Trim();
-            var value = parts[i + 2].Trim().Trim('\'');
 
-            if (value.Equals("NULL", StringComparison.OrdinalIgnoreCase))
+            bool expr;
+            int consumed;
+
+            // Handle IS NOT NULL (4 tokens: key IS NOT NULL)
+            if (op.Equals("IS", StringComparison.OrdinalIgnoreCase)
+                && i + 3 < parts.Length
+                && parts[i + 2].Trim().Equals("NOT", StringComparison.OrdinalIgnoreCase)
+                && parts[i + 3].Trim().Equals("NULL", StringComparison.OrdinalIgnoreCase))
             {
-                value = null;
+                expr = row.TryGetValue(key, out var v) && v is not null && v is not DBNull;
+                consumed = 4;
             }
-
-            bool expr = false;
-            if (row.ContainsKey(key))
+            // Handle IS NULL (3 tokens: key IS NULL)
+            else if (op.Equals("IS", StringComparison.OrdinalIgnoreCase)
+                     && parts[i + 2].Trim().Equals("NULL", StringComparison.OrdinalIgnoreCase))
             {
-                var rowValue = row[key];
-                expr = EvaluateOperator(rowValue, op, value);
+                expr = !row.TryGetValue(key, out var v) || v is null || v is DBNull;
+                consumed = 3;
+            }
+            else
+            {
+                var value = parts[i + 2].Trim().Trim('\'');
+
+                if (value.Equals("NULL", StringComparison.OrdinalIgnoreCase))
+                {
+                    value = null;
+                }
+
+                expr = false;
+                if (row.ContainsKey(key))
+                {
+                    var rowValue = row[key];
+                    expr = EvaluateOperator(rowValue, op, value);
+                }
+                consumed = 3;
             }
 
             if (current is null)
@@ -320,7 +362,7 @@ private static bool EvaluateComplexCondition(Dictionary<string, object> row, str
             }
 
             // Move index to next token after this expression
-            i += 3;
+            i += consumed;
             // Read logical connector if present
             if (i < parts.Length)
             {
diff --git a/src/SharpCoreDB/Services/SqlParser.PerformanceOptimizations.cs b/src/SharpCoreDB/Services/SqlParser.PerformanceOptimizations.cs
@@ -244,14 +244,26 @@ private static Func<Dictionary<string, object>, bool> CompilePredicateFromParts(
     private static Func<Dictionary<string, object>, bool> CompileSingleCondition(string condition)
     {
         condition = condition.Trim();
-        
+
+        // Check for IS NULL / IS NOT NULL first (no right-hand value)
+        var isNullPattern = new Regex(@"^(\w+)\s+IS\s+(NOT\s+)?NULL\s*$", RegexOptions.IgnoreCase);
+        var isNullMatch = isNullPattern.Match(condition);
+        if (isNullMatch.Success)
+        {
+            string col = isNullMatch.Groups[1].Value.Trim();
+            bool isNotNull = isNullMatch.Groups[2].Success;
+            return isNotNull
+                ? row => row.TryGetValue(col, out var val) && val is not null && val is not DBNull
+                : row => !row.TryGetValue(col, out var val) || val is null || val is DBNull;
+        }
+
         // Try to match: column operator value
         // Operators: =, !=, >, <, >=, <=, IN, LIKE
-        
+
         var operatorPattern = new Regex(@"(\w+)\s*(=|!=|>=|<=|>|<|IN|LIKE)\s*(.+)", 
             RegexOptions.IgnoreCase);
         var match = operatorPattern.Match(condition);
-        
+
         if (!match.Success)
         {
             // Can't parse, accept all
diff --git a/src/SharpCoreDB/Storage/SingleFileStorageProvider.cs b/src/SharpCoreDB/Storage/SingleFileStorageProvider.cs
@@ -1249,7 +1249,10 @@ public void Dispose()
                 RollbackTransaction();
             }
 
-            // ✅ Phase 1 Task 1.3: Flush write-behind queue before shutdown
+            // Complete the write queue first so the worker can drain remaining items
+            _writeQueue.Writer.TryComplete();
+
+            // Cancel the write worker and wait for it to finish draining
             _writeCts.Cancel();
             try
             {
@@ -1259,18 +1262,29 @@ public void Dispose()
             {
                 // Expected
             }
+            catch
+            {
+                // Worker may fail during shutdown — best effort
+            }
 
-            ForceFlushAsync().GetAwaiter().GetResult();
+            // Force flush any remaining pending changes (guard against disposed subsystems)
+            try
+            {
+                ForceFlushAsync(CancellationToken.None).GetAwaiter().GetResult();
+            }
+            catch
+            {
+                // Best effort — subsystems may already be partially torn down
+            }
 
             _blockRegistry?.Dispose();
             _freeSpaceManager?.Dispose();
             _walManager?.Dispose();
             _tableDirectoryManager?.Dispose();
             _memoryMappedFile?.Dispose();
             _fileStream?.Dispose();
-            
-            // ✅ Cleanup write queue resources
-            _writeQueue.Writer.Complete();
+
+            // Cleanup write queue resources
             _writeCts?.Dispose();
             _flushSignal?.Dispose();
         }
diff --git a/tests/SharpCoreDB.Tests/EngineLimitationFixTests.cs b/tests/SharpCoreDB.Tests/EngineLimitationFixTests.cs
diff --git a/tests/SharpCoreDB.Tests/Phase9_LocaleCollationsTests.cs b/tests/SharpCoreDB.Tests/Phase9_LocaleCollationsTests.cs
diff --git a/tests/SharpCoreDB.Tests/SingleFileTests.cs b/tests/SharpCoreDB.Tests/SingleFileTests.cs
diff --git a/tmp_src_default_stub_scan.txt b/tmp_src_default_stub_scan.txt

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ public partial class EnhancedSqlParser(ISqlDialect? dialect = null)`
`50`	`50`	`{`
`51`	`51`	`var keyword = PeekKeyword();`
`52`	`52`
`53`		`- return keyword?.ToUpperInvariant() switch`
	`53`	`+ var result = keyword?.ToUpperInvariant() switch`
`54`	`54`	`{`
`55`	`55`	`"SELECT" => ParseSelect(),`
`56`	`56`	`"INSERT" => ParseInsert(),`
`@@ -60,6 +60,15 @@ public partial class EnhancedSqlParser(ISqlDialect? dialect = null)`
`60`	`60`	`"ALTER" => ParseAlter(),`
`61`	`61`	`_ => throw new InvalidOperationException($"Unsupported statement type: {keyword}")`
`62`	`62`	`};`
	`63`	`+`
	`64`	`+ // Validate no unparsed trailing tokens remain`
	`65`	`+ var remaining = _sql.Substring(_position).Trim();`
	`66`	`+ if (remaining.Length > 0)`
	`67`	`+ {`
	`68`	`+ RecordError($"Unexpected trailing content: '{remaining}'");`
	`69`	`+ }`
	`70`	`+`
	`71`	`+ return result;`
`63`	`72`	`}`
`64`	`73`	`catch (Exception ex)`
`65`	`74`	`{`