push all current changes for SQLITE / Postgre Dialect and the new Functional SQL engine

Author: MPCoreDeveloper

.github/copilot-instructions.md

@@ -15,7 +15,7 @@
 - Use `xunit.runner.visualstudio` 3.1.5+ for test discovery.
 - If you encounter any project referencing `xunit` (without `.v3`), migrate it to `xunit.v3` immediately.
 - Test runner: `Microsoft.NET.Test.Sdk` 18.3.0+ (latest stable for .NET 10).
-- Prefer targeted, fast test runs instead of broad/long-running full-suite test execution during iterative work.
+- Prefer targeted, fast test runs instead of broad/long-running full-suite test execution during iterative work. Query failed tests directly instead of performing broad test scans, especially when a single hanging test is the bottleneck.
 
 ## Code Style
 - Use specific formatting rules.

Examples/Web/Orchardcore/SharpCoreDb.Orchardcore/SharpCoreDb.Orchardcore.csproj

@@ -21,10 +21,11 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="HtmlSanitizer" Version="9.0.892" />
-    <PackageReference Include="MimeKit" Version="4.15.1" />
+    <PackageReference Include="MailKit" Version="4.16.0" />
+    <PackageReference Include="MimeKit" Version="4.16.0" />
     <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="10.5.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="10.0.6" />
-    <PackageReference Include="AWSSDK.Core" Version="4.0.3.29" />
+    <PackageReference Include="AWSSDK.Core" Version="4.0.3.30" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(UseLocalSharpCoreDbSources)' == 'true'">

SharpCoreDB.sln

@@ -1,3 +1,4 @@
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 18
 VisualStudioVersion = 18.1.11312.151
@@ -211,6 +212,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpCoreDB.Identity", "src
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpCoreDB.CrudApp", "Examples\Web\SharpCoreDB.CrudApp\SharpCoreDB.CrudApp.csproj", "{FED5C1D6-0A80-DD7D-6C55-1AD989A759CA}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpCoreDB.Functional.Tests", "tests\SharpCoreDB.Functional.Tests\SharpCoreDB.Functional.Tests.csproj", "{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -869,6 +872,18 @@ Global
 		{FED5C1D6-0A80-DD7D-6C55-1AD989A759CA}.Release|x64.Build.0 = Release|Any CPU
 		{FED5C1D6-0A80-DD7D-6C55-1AD989A759CA}.Release|x86.ActiveCfg = Release|Any CPU
 		{FED5C1D6-0A80-DD7D-6C55-1AD989A759CA}.Release|x86.Build.0 = Release|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Debug|x64.Build.0 = Debug|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Debug|x86.Build.0 = Debug|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Release|x64.ActiveCfg = Release|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Release|x64.Build.0 = Release|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Release|x86.ActiveCfg = Release|Any CPU
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -942,6 +957,7 @@ Global
 		{DC85D720-0F4B-4904-8369-753A9365765F} = {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}
 		{09550D6E-8D1A-F43A-0C06-B03F7A2A6193} = {F8B5E3A4-1C2D-4E5F-8B9A-1D2E3F4A5B6C}
 		{FED5C1D6-0A80-DD7D-6C55-1AD989A759CA} = {50B921FA-F081-4B7B-A8E0-AF3DCC098CCC}
+		{87E3DF4B-8BF3-45CC-85A0-218418ED54AA} = {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F40825F5-26A1-4E85-9D0A-B0121A7ED5F8}

SharpCoreDB.slnx

@@ -124,6 +124,7 @@
     <Project Path="tests/SharpCoreDB.DemoJoinsSubQ/SharpCoreDB.DemoJoinsSubQ.csproj" />
     <Project Path="tests/SharpCoreDB.EntityFrameworkCore.Tests/SharpCoreDB.EntityFrameworkCore.Tests.csproj" />
     <Project Path="tests/SharpCoreDB.EventSourcing.Tests/SharpCoreDB.EventSourcing.Tests.csproj" />
+    <Project Path="tests/SharpCoreDB.Functional.Tests/SharpCoreDB.Functional.Tests.csproj" />
     <Project Path="tests/SharpCoreDB.Profiling/SharpCoreDB.Profiling.csproj" />
     <Project Path="tests/SharpCoreDB.Projections.Tests/SharpCoreDB.Projections.Tests.csproj" />
     <Project Path="tests/SharpCoreDB.Provider.Sync.Tests/SharpCoreDB.Provider.Sync.Tests.csproj" />

docs/CHANGELOG.md

@@ -16,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `ColumnInfo.IsHidden` property for metadata-driven schema tools.
 - `PersistenceConstants.InternalRowIdColumnName` constant (`"_rowid"`).
 - 9 dedicated tests for the Auto-ROWID feature in `AutoRowIdTests.cs`.
+- **GRAPH_RAG SQL clause**: New top-level `GRAPH_RAG` SELECT clause with `LIMIT`, `WITH SCORE > X`, `WITH CONTEXT`, and `TOP_K` options, plus provider-based execution integration via `IGraphRagProvider`.
+- **OPTIONALLY SQL projection mode**: New `OPTIONALLY` keyword after SELECT list enabling `Option<T>` mapping in ADO.NET readers, integrated with `SharpCoreDB.Functional`.
+- **SOME/NONE predicates**: New `IS SOME` and `IS NONE` predicates (and NOT variants) supported in parser and runtime evaluators.
 
 ### Fixed
 - Unified `IS NULL` / `IS NOT NULL` behavior across runtime scan, join-helper, and compiled predicate paths.

docs/FEATURE_MATRIX_v1.7.0.md

@@ -6,7 +6,7 @@ This page consolidates the major SharpCoreDB capabilities by package for quick d
 
 | Package | Purpose | Key capabilities in v1.7.0 |
 |---|---|---|
-| `SharpCoreDB` | Embedded core engine | AES-256-GCM encryption, SQL engine, ACID + WAL, indexing, FTS, SIMD optimizations, metadata durability fixes, compiled-query parser fixes |
+| `SharpCoreDB` | Embedded core engine | AES-256-GCM encryption, SQL engine, ACID + WAL, indexing, FTS, SIMD optimizations, metadata durability fixes, compiled-query parser fixes, `GRAPH_RAG` SQL clause, `OPTIONALLY` + `IS SOME`/`IS NONE` optional SQL semantics |
 | `SharpCoreDB.Server` | Network server runtime | gRPC-first (HTTP/2 + HTTP/3), REST, WebSocket, JWT/RBAC, optional mTLS, multi-database hosting, health/metrics |
 | `SharpCoreDB.Client` | .NET client | ADO.NET-style commands/readers, async access, parameterized execution, server connectivity |
 

docs/README.md

@@ -13,6 +13,7 @@ This folder contains the maintained documentation set for SharpCoreDB (`v1.7.0`)
 ## Core Areas
 
 - `server/` - Server operation, security, APIs, and protocols.
+- `sql/` - SQL dialect capabilities, extensions, and compatibility guidance.
 - `scdb/` - Storage engine and SCDB guidance.
 - `serialization/` - Binary format and serialization internals.
 - `analytics/` - Analytics capabilities and usage.
@@ -22,6 +23,12 @@ This folder contains the maintained documentation set for SharpCoreDB (`v1.7.0`)
 - `sync/` - Dotmim.Sync provider usage.
 - `migration/` - Migration and interoperability guides.
 
+## SQL extension docs (v1.7.0)
+
+- `sql/SQL_DIALECT_EXTENSIONS_v1.7.0.md` - SharpCoreDB-specific SQL extensions including `GRAPH_RAG`, `OPTIONALLY`, and `IS SOME`/`IS NONE`.
+- `graphrag/GRAPH_RAG_SINGLE_SQL.md` - Single-statement GraphRAG SQL syntax and DI integration.
+- `functional/OPTIONALLY_SQL_OPTION_SUPPORT_v1.7.0.md` - Option<T> mapping semantics and usage patterns.
+
 ## Package Documentation
 
 Per-package docs are maintained in `src/*/README.md` and `src/*/NuGet.README.md`, aligned to `v1.7.0`.

docs/compatibility/SQLITE_POSTGRESQL_AGGREGATE_SYNTAX_v1.7.0.md

@@ -98,7 +98,7 @@
 | `BETWEEN … AND …` | ✅ | ✅ | ✅ | |
 | `IN (…)` / `NOT IN (…)` | ✅ | ✅ | ✅ | |
 | `LIKE` (% and _ wildcards) | ✅ | ✅ | ✅ | **New in v1.7.0** — scalar function-form `LIKE(value, pattern)` supported in literal SELECT path |
-| `GLOB` | ✅ | ❌ | ❌ | Gap — SQLite-specific |
+| `GLOB` | ✅ | ❌ | ✅ | Implemented v1.7.0 |
 | `REGEXP` | ✅ extension | ✅ | ✅ | **New in v1.7.0** — AST WHERE evaluation supports `REGEXP` / `NOT REGEXP` |
 | `ORDER BY col [ASC\|DESC]` | ✅ | ✅ | ✅ | |
 | `ORDER BY ordinal position` (e.g. `ORDER BY 2`) | ✅ | ✅ | ✅ | |
@@ -118,13 +118,13 @@
 | `WITH RECURSIVE` CTE | ✅ | ✅ | ❌ | Gap |
 | `CASE WHEN … THEN … ELSE … END` | ✅ | ✅ | ⚠️ | Evaluated in AST executor; not in legacy DML path |
 | `CAST(… AS type)` | ✅ | ✅ | ⚠️ | TypeConverter handles common casts |
-| `COALESCE(…)` | ✅ | ✅ | ❌ | Gap |
-| `IFNULL(a, b)` / `NULLIF(a, b)` | ✅ | ✅ | ❌ | Gap |
-| `IIF(cond, t, f)` | ✅ | ❌ | ❌ | Gap |
-| `TYPEOF(x)` | ✅ | ❌ | ❌ | Gap |
+| `COALESCE(…)` | ✅ | ✅ | ✅ | **New in v1.7.0** — AST scalar function dispatch |
+| `IFNULL(a, b)` / `NULLIF(a, b)` | ✅ | ✅ | ✅ | **New in v1.7.0** — AST scalar function dispatch |
+| `IIF(cond, t, f)` | ✅ | ❌ | ✅ | **New in v1.7.0** — AST scalar function dispatch |
+| `TYPEOF(x)` | ✅ | ❌ | ✅ | **New in v1.7.0** — SQLite type names (`integer`, `real`, `text`, `blob`, `null`) |
 | `EXISTS (subquery)` / `NOT EXISTS` | ✅ | ✅ | ⚠️ | Partial |
-| `UNION` / `UNION ALL` | ✅ | ✅ | ❌ | Gap |
-| `INTERSECT` / `EXCEPT` | ✅ | ✅ | ❌ | Gap |
+| `UNION` / `UNION ALL` | ✅ | ✅ | ✅ | Implemented v1.7.0 |
+| `INTERSECT` / `EXCEPT` | ✅ | ✅ | ✅ | Implemented v1.7.0 |
 | `EXPLAIN` / `EXPLAIN QUERY PLAN` | ✅ | ✅ | ⚠️ | Text output; not structured rows |
 
 ---
@@ -181,10 +181,10 @@
 | `TRIM(s)` / `LTRIM(s)` / `RTRIM(s)` | ✅ | ✅ | ✅ | **New in v1.7.0** — AST scalar function dispatch |
 | `REPLACE(s, from, to)` | ✅ | ✅ | ✅ | **New in v1.7.0** — AST scalar function dispatch |
 | `INSTR(s, sub)` | ✅ | ❌ | ✅ | **New in v1.7.0** — SQLite-compatible 1-based index |
-| `LIKE(s, pattern)` | ✅ | ✅ | ❌ | Gap — function form |
+| `LIKE(s, pattern)` | ✅ | ✅ | ✅ | **New in v1.7.0** — scalar function-form supported |
 | `HEX(x)` / `UNHEX(x)` | ✅ | ❌ | ✅ | `HEX(x)` and `UNHEX(x)` both supported |
 | `QUOTE(x)` | ✅ | ❌ | ✅ | **New in v1.7.0** — returns SQL literal text with proper single-quote escaping |
-| `RANDOM()` | ✅ | ✅ | ❌ | Gap |
+| `RANDOM()` | ✅ | ✅ | ✅ | Implemented v1.7.0 |
 | `COALESCE(a, b, …)` | ✅ | ✅ | ✅ | **New in v1.7.0** — AST scalar function dispatch |
 | `IFNULL(a, b)` | ✅ | ❌ | ✅ | **New in v1.7.0** — SQLite alias of COALESCE |
 | `NULLIF(a, b)` | ✅ | ✅ | ✅ | **New in v1.7.0** — AST scalar function dispatch |
@@ -266,11 +266,11 @@ These are the most impactful missing features for full SQLite compatibility, ord
 |---|---|---|
 | 🔴 P0 | `COALESCE` / `IFNULL` scalar functions | Missing scalar function |
 | 🔴 P0 | `INSERT OR REPLACE` / `INSERT OR IGNORE` | Missing DML variant |
-| 🟠 P1 | `UNION` / `UNION ALL` / `INTERSECT` / `EXCEPT` | Missing set operations |
+| ✅ Done | `UNION` / `UNION ALL` / `INTERSECT` / `EXCEPT` | Implemented v1.7.0 |
 | 🟠 P1 | `WITH RECURSIVE` (recursive CTE) | Missing CTE variant |
-| 🟠 P1 | `GLOB` operator | Missing WHERE operator |
+| ✅ Done | `GLOB` operator | Implemented v1.7.0 |
 | 🟠 P1 | String functions: `LIKE()` function-form, `UNHEX`, `QUOTE`, `CHAR`, `UNICODE` | Missing scalar functions |
-| 🟠 P1 | Numeric functions: scalar `MAX/MIN`, `POW/POWER`, `SQRT`, `MOD`, `RANDOM` | Missing scalar functions |
+| ✅ Done | Numeric functions: scalar `MAX/MIN`, `POW/POWER`, `SQRT`, `MOD`, `RANDOM` | Implemented v1.7.0 |
 | 🟡 P2 | `CHANGES()` / `TOTAL_CHANGES()` full semantics | Missing scalar function semantics |
 | 🟡 P2 | `CHECK` constraint enforcement at DML time | Constraint enforcement |
 | 🟡 P2 | `SAVEPOINT` / `RELEASE` / `ROLLBACK TO` | Missing transaction feature |

docs/compatibility/SQLITE_POSTGRESQL_COMPATIBILITY_TODO.md

@@ -2,8 +2,7 @@
 
 > **Goal:** Full SQLite syntax parity + key PostgreSQL additions in SharpCoreDB.  
 > **Reference matrix:** `docs/compatibility/SQLITE_POSTGRESQL_AGGREGATE_SYNTAX_v1.7.0.md`  
-> **Last updated:** v1.7.0  
-> **Estimated total effort:** ~19 weeks solo / ~10 weeks with AI pair-programming.
+> **Last updated:** v1.7.0 — 2025-07-03 (W3-5 DEFAULT expression marked done; AstExecutor VisitSetOperation implemented)
 
 ---
 
@@ -81,17 +80,17 @@
 ## 🟠 Wave 2 — P1 High Impact (~8 weeks)
 
 ### W2-1 · Set operations
-- [ ] Parse `UNION` between two SELECT arms
-- [ ] Parse `UNION ALL`
-- [ ] Parse `INTERSECT`
-- [ ] Parse `EXCEPT`
-- [ ] Add `SetOperationNode` to `SqlAst.Nodes.cs`
-- [ ] Execute UNION (deduplicate)
-- [ ] Execute UNION ALL (no dedup)
-- [ ] Execute INTERSECT
-- [ ] Execute EXCEPT
-- [ ] `ORDER BY` / `LIMIT` on outer set result
-- [ ] Tests for all four set operations
+- [x] Parse `UNION` between two SELECT arms
+- [x] Parse `UNION ALL`
+- [x] Parse `INTERSECT`
+- [x] Parse `EXCEPT`
+- [x] Add `SetOperationNode` to `SqlAst.Nodes.cs`
+- [x] Execute UNION (deduplicate)
+- [x] Execute UNION ALL (no dedup)
+- [x] Execute INTERSECT
+- [x] Execute EXCEPT
+- [x] `ORDER BY` / `LIMIT` on outer set result
+- [x] Tests for all four set operations
 
 ### W2-2 · String scalar functions (extended)
 - [x] `SUBSTR(s, start)` — 1-based index, SQLite convention
@@ -107,20 +106,20 @@
 - [x] Tests for implemented string functions
 
 ### W2-3 · Numeric scalar functions (extended)
-- [ ] `MAX(a, b)` — 2-arg scalar form (distinct from aggregate MAX)
-- [ ] `MIN(a, b)` — 2-arg scalar form
-- [ ] `POW(x, y)` / `POWER(x, y)`
-- [ ] `SQRT(x)`
-- [ ] `MOD(x, y)` / `%` operator
+- [x] `MAX(a, b)` — 2-arg scalar form (distinct from aggregate MAX)
+- [x] `MIN(a, b)` — 2-arg scalar form
+- [x] `POW(x, y)` / `POWER(x, y)`
+- [x] `SQRT(x)`
+- [x] `MOD(x, y)` / `%` operator
 - [x] `SIGN(x)`
-- [ ] `RANDOM()` — returns random integer (SQLite: 64-bit signed)
+- [x] `RANDOM()` — returns random integer (SQLite: 64-bit signed)
 - [x] Tests for implemented numeric functions
 
 ### W2-4 · GLOB operator
-- [ ] Parse `col GLOB pattern` in expression parser
-- [ ] Implement glob→regex conversion (`*`→`.*`, `?`→`.`, case-sensitive)
-- [ ] Support character classes `[A-Z]`
-- [ ] Tests for GLOB with various patterns
+- [x] Parse `col GLOB pattern` in expression parser
+- [x] Implement glob→regex conversion (`*`→`.*`, `?`→`.`, case-sensitive)
+- [x] Support character classes `[A-Z]`
+- [x] Tests for GLOB with various patterns
 
 ### W2-5 · `WITH RECURSIVE` CTE
 - [ ] Detect `RECURSIVE` keyword in `WITH` clause
@@ -171,9 +170,9 @@
 - [ ] `LAST_INSERT_ROWID()` — verify works in all execution paths
 
 ### W3-5 · DEFAULT expression evaluation
-- [ ] At INSERT time, evaluate `DEFAULT (expr)` for omitted columns
-- [ ] Support `DEFAULT (strftime('%Y-%m-%d', 'now'))`
-- [ ] Tests for DEFAULT expression columns
+- [x] At INSERT time, evaluate `DEFAULT (expr)` for omitted columns
+- [x] Support `DEFAULT (strftime('%Y-%m-%d', 'now'))`
+- [x] Tests for DEFAULT expression columns
 
 ### W3-6 · Correlated subquery (full support)
 - [ ] Pass outer row context into inner executor on each iteration
@@ -242,11 +241,11 @@
 | Wave | Total items | Done | Remaining |
 |---|---|---|---|
 | Quick Wins | 14 | 14 | 0 |
-| W1 — P0 | 18 | 2 | 16 |
-| W2 — P1 | 34 | 8 | 26 |
-| W3 — P2 | 28 | 2 | 26 |
-| W4 — P3 | 24 | 1 | 23 |
-| **Total** | **118** | **27** | **91** |
+| W1 — P0 | 31 | 31 | 0 |
+| W2 — P1 | 42 | 35 | 7 |
+| W3 — P2 | 38 | 5 | 33 |
+| W4 — P3 | 27 | 1 | 26 |
+| **Total** | **152** | **86** | **66** |
 
 ---
 

docs/functional/OPTIONALLY_SQL_OPTION_SUPPORT_v1.7.0.md

@@ -0,0 +1,58 @@
+# OPTIONALLY SQL + Option<T> support (v1.7.0)
+
+SharpCoreDB now supports native optional-value query shaping for functional applications.
+
+## SQL syntax
+
+```sql
+SELECT id, name, email OPTIONALLY
+FROM users
+WHERE email IS SOME;
+```
+
+Supported predicates:
+
+- `IS SOME` → matches non-null values
+- `IS NONE` → matches null/DBNull values
+
+## Behavior
+
+When `OPTIONALLY` is used in `SELECT`, the ADO.NET provider maps projected values to `Option<T>` from `SharpCoreDB.Functional`:
+
+- Non-null value → `Option.Some<T>(value)`
+- Null value → `Option.None<T>()`
+
+Type inference rules:
+
+- Uses first non-null value in each column to infer inner `T`
+- If all values are null, defaults to `Option<object>`
+
+## CQRS / Event Sourcing advantage
+
+`OPTIONALLY` reduces defensive null-checks in handlers and projections.
+
+### Without OPTIONALLY
+
+```csharp
+var emailObj = reader["email"];
+if (emailObj is null || emailObj is DBNull)
+{
+    // fallback path
+}
+else
+{
+    var email = emailObj.ToString();
+    // normal path
+}
+```
+
+### With OPTIONALLY
+
+```csharp
+var emailOpt = (Option<string>)reader["email"];
+var normalized = emailOpt.Match(
+    Some: email => email.Trim().ToLowerInvariant(),
+    None: () => "(missing)");
+```
+
+This aligns naturally with functional pipelines used in CQRS command handlers, read models, and event upcasters.