Merge pull request #144 from NodeDB-Lab/feature/temporal-queries

feat: add temporal query support across engines

Author: farhan-syah

.github/workflows/release.yml

@@ -106,7 +106,10 @@ jobs:
             # For pre-release versions, pin internal dep requirements so semver
             # "0.0.0" doesn't fail to match "0.0.0-beta.1"
             if [[ "$VERSION" == *-* ]]; then
-              sed -i -E 's/(nodedb-(codec|bridge|wal|mem|crdt|raft|fts|types|vector|graph|spatial|strict|client|columnar|cluster|query|sql|array|physical) = \{ [^}]*version = )"[^"]*"/\1"='"$VERSION"'"/' Cargo.toml
+              # Pin every internal nodedb* path-dep (bare `nodedb` included) to the
+              # exact prerelease version. A generic match avoids the recurring bug
+              # of forgetting to add a newly-introduced crate to an explicit list.
+              sed -i -E 's/(nodedb[a-z0-9-]* = \{ [^}]*version = )"[^"]*"/\1"='"$VERSION"'"/' Cargo.toml
               echo "Updated internal dep versions to =$VERSION"
             fi
 
@@ -233,7 +236,10 @@ jobs:
             # For pre-release versions, pin internal dep requirements so semver
             # "0" doesn't fail to match "0.0.0-beta.3"
             if [[ "$VERSION" == *-* ]]; then
-              sed -i -E 's/(nodedb-(codec|bridge|wal|mem|crdt|raft|fts|types|vector|graph|spatial|strict|client|columnar|cluster|query|sql|array|physical) = \{ [^}]*version = )"[^"]*"/\1"='"$VERSION"'"/' Cargo.toml
+              # Pin every internal nodedb* path-dep (bare `nodedb` included) to the
+              # exact prerelease version. A generic match avoids the recurring bug
+              # of forgetting to add a newly-introduced crate to an explicit list.
+              sed -i -E 's/(nodedb[a-z0-9-]* = \{ [^}]*version = )"[^"]*"/\1"='"$VERSION"'"/' Cargo.toml
               echo "Updated internal dep versions to =$VERSION"
             fi
 

.gitignore

@@ -1,5 +1,6 @@
 # Rust build artifacts
 /target/
+/releases/
 
 # Data directories (fallback default or manual testing)
 /data/

Cargo.lock

@@ -22,7 +22,7 @@ NodeDB splits work across three planes connected by lock-free ring buffers. This
 └────────────────────────┘  └─────────────────────────────────────┘
 ```
 
-**Control Plane** — Runs on Tokio. Handles connections (pgwire, HTTP, WebSocket), parses SQL via DataFusion, builds logical query plans, and dispatches work to the Data Plane. All types here are `Send + Sync`.
+**Control Plane** — Runs on Tokio. Handles connections (pgwire, HTTP, WebSocket), parses SQL via nodedb-sql (sqlparser-rs based), builds logical query plans, and dispatches work to the Data Plane. All types here are `Send + Sync`.
 
 **Data Plane** — One thread per CPU core, each an isolated shard. Reads from NVMe via io_uring, runs SIMD vector math, executes physical query plans. No locks, no atomics, no cross-core sharing. Types are `!Send` by design. Emits `WriteEvent` records (covering inserts, updates, and deletes via `WriteOp`) to the Event Plane via per-core bounded ring buffers after each WAL commit.
 
@@ -44,13 +44,13 @@ NodeDB splits work across three planes connected by lock-free ring buffers. This
 
 There are two ways a query reaches the Data Plane. Both produce the same `PhysicalPlan` and execute identically from that point on.
 
-**SQL path (user-facing)** — All user-visible interfaces use SQL. `psql`, the `ndb` CLI, and the HTTP `/v1/query` endpoint all accept SQL text. The Control Plane runs it through DataFusion (parse → logical plan → optimize → `PhysicalPlan`):
+**SQL path (user-facing)** — All user-visible interfaces use SQL. `psql`, the `ndb` CLI, and the HTTP `/v1/query` endpoint all accept SQL text. The Control Plane runs it through nodedb-sql (parse → logical plan → optimize → `PhysicalPlan`):
 
 ```
 psql / ndb CLI / HTTP /v1/query
          │
          ▼
-   DataFusion parser
+   nodedb-sql parser
          │
          ▼
    Logical plan + optimizer
@@ -59,7 +59,7 @@ psql / ndb CLI / HTTP /v1/query
    PhysicalPlan ──► SPSC Bridge ──► Data Plane
 ```
 
-**Native opcode path (SDK optimization)** — The Rust SDK (`nodedb-client`), FFI bindings (`nodedb-lite-ffi`), and WASM bindings (`nodedb-lite-wasm`) dispatch typed opcode messages over the NDB protocol instead of SQL text. The Control Plane converts them directly to a `PhysicalPlan` via `build_plan()`, skipping SQL parsing and serialization:
+**Native opcode path (SDK optimization)** — The Rust SDK (`nodedb-client`), FFI bindings (`nodedb-lite-ffi`), and WASM bindings (`nodedb-lite-wasm`) dispatch typed opcode messages over the NDB protocol instead of SQL text. The Control Plane converts them directly to a `PhysicalPlan` via engine-specific handlers, skipping SQL parsing and serialization:
 
 ```
 nodedb-client / nodedb-lite-ffi / nodedb-lite-wasm

docs/architecture.md

@@ -29,18 +29,18 @@ Array schemas are defined by dimensions (axes), attributes (stored values), and
 ```sql
 CREATE ARRAY spatial_grid
   DIMS (
-    x INT64 DOMAIN [0, 1000),
-    y INT64 DOMAIN [0, 1000),
-    z INT64 DOMAIN [0, 1000)
+    x INT64 [0..1000],
+    y INT64 [0..1000],
+    z INT64 [0..1000]
   )
   ATTRS (
-    temperature FLOAT32,
-    pressure FLOAT32,
-    humidity FLOAT32
+    temperature FLOAT64,
+    pressure FLOAT64,
+    humidity FLOAT64
   )
   TILE_EXTENTS (64, 64, 64)
   WITH (
-    cell_order = 'Z-ORDER',
+    prefix_bits = 8,
     audit_retain_ms = 86400000
   );
 ```
@@ -49,11 +49,14 @@ CREATE ARRAY spatial_grid
 
 | Parameter         | Required | Default     | Description                                                                                                        |
 | ----------------- | -------- | ----------- | ------------------------------------------------------------------------------------------------------------------ |
-| `DIMS`            | Yes      | —           | List of dimensions. Each has a name, type (`INT64`, `INT32`, `FLOAT64`), and domain bounds `[lo, hi)`.             |
-| `ATTRS`           | Yes      | —           | List of attributes (cell values). Each has a name and type (`FLOAT32`, FLOAT64`, `INT32`, `INT64`, `STRING`).      |
+| `DIMS`            | Yes      | —           | List of dimensions. Each has a name, type (`INT64`, `FLOAT64`, `TIMESTAMP_MS`, `STRING`), and optional domain bounds `[lo..hi]`. |
+| `ATTRS`           | Yes      | —           | List of attributes (cell values). Each has a name and type (`INT64`, `FLOAT64`, `STRING`, `BYTES`).               |
 | `TILE_EXTENTS`    | Yes      | —           | Tuple of tile extent per dimension. All > 0. Determines cell locality and compression block granularity.           |
-| `cell_order`      | No       | `'Z-ORDER'` | `'Z-ORDER'` (Hilbert curve) or `'ROW-MAJOR'`. Affects spatial cache locality.                                      |
+| `CELL_ORDER`      | No       | `Z_ORDER`   | `ROW_MAJOR`, `COL_MAJOR`, `Z_ORDER`, or `HILBERT`. Affects spatial cache locality.                                 |
+| `TILE_ORDER`      | No       | —           | `ROW_MAJOR`, `COL_MAJOR`, `Z_ORDER`, or `HILBERT`. Optional tile ordering.                                         |
+| `prefix_bits`     | No       | `8`         | Bits for key prefix compression (1–16). Lower = better compression, higher = faster prefix filtering.              |
 | `audit_retain_ms` | No       | `NULL`      | Milliseconds. Tiles older than now - `audit_retain_ms` (by system time) are eligible for purge. `NULL` = keep all. |
+| `minimum_audit_retain_ms` | No | —        | Minimum retention window enforced for compliance.                                                                  |
 
 ## Examples
 
@@ -63,11 +66,11 @@ CREATE ARRAY spatial_grid
 -- Create a 2D elevation map with tiles of 256x256 cells
 CREATE ARRAY elevation_map
   DIMS (
-    lon FLOAT64 DOMAIN [-180, 180),
-    lat FLOAT64 DOMAIN [-90, 90)
+    lon FLOAT64 [-180..180],
+    lat FLOAT64 [-90..90]
   )
   ATTRS (
-    height FLOAT32
+    height FLOAT64
   )
   TILE_EXTENTS (256, 256);
 
@@ -88,13 +91,13 @@ SELECT ARRAY_FLUSH('elevation_map');
 -- Create a 3D climate model with temporal tracking
 CREATE ARRAY climate_forecast
   DIMS (
-    lon INT32 DOMAIN [-180, 180),
-    lat INT32 DOMAIN [-90, 90),
-    altitude_m INT32 DOMAIN [0, 50000)
+    lon INT64 [-180..180],
+    lat INT64 [-90..90],
+    altitude_m INT64 [0..50000]
   )
   ATTRS (
-    temp_c FLOAT32,
-    humidity FLOAT32
+    temp_c FLOAT64,
+    humidity FLOAT64
   )
   TILE_EXTENTS (32, 32, 20)
   WITH (audit_retain_ms = 7776000000);  -- 90 days

docs/array.md

@@ -40,7 +40,10 @@ AS OF SYSTEM TIME 1700000000000
 AS OF VALID TIME 1700000001000;
 ```
 
-Times are milliseconds since Unix epoch. Use `extract(epoch from now()) * 1000` for current time.
+Times can be specified as:
+- Integer milliseconds since Unix epoch (e.g., `1700000000000`)
+- `NOW()` for current time
+- ISO-8601 string literal (e.g., `'2024-01-15T00:00:00Z'`)
 
 ## Examples
 
@@ -50,12 +53,12 @@ Retrieve the state of a collection at a past moment:
 
 ```sql
 -- Document collection with system time tracking
-CREATE COLLECTION user_accounts STRICT (
+CREATE COLLECTION user_accounts (
     id UUID DEFAULT gen_uuid_v7(),
     email VARCHAR,
     balance DECIMAL,
     created_at TIMESTAMP DEFAULT now()
-);
+) WITH (engine='document_strict', bitemporal=true);
 
 INSERT INTO user_accounts (email, balance) VALUES
     ('alice@example.com', 100.00);
@@ -65,7 +68,7 @@ UPDATE user_accounts SET balance = 150.00 WHERE email = 'alice@example.com';
 
 -- Query the database as it existed 10 minutes ago
 SELECT email, balance FROM user_accounts
-AS OF SYSTEM TIME (extract(epoch from now()) * 1000 - 600000);
+AS OF SYSTEM TIME 1700000000000;
 -- Returns: alice@example.com, 100.00 (the original balance)
 
 -- Query current state
@@ -96,7 +99,7 @@ VALUES ('2026-04-01T10:00:00Z', 'warehouse-a', 22.3, '2026-04-02T15:30:00Z');
 -- Query what we knew at April 1st (before correction)
 SELECT location, temperature FROM sensor_readings
 WHERE ts BETWEEN '2026-04-01' AND '2026-04-02'
-AS OF VALID TIME 1711953600000;  -- April 1st
+AS OF VALID TIME '2026-04-01T00:00:00Z';
 
 -- Query what we know now (after correction)
 SELECT location, temperature FROM sensor_readings
@@ -127,7 +130,7 @@ SELECT lon, lat, temp_c FROM ARRAY_SLICE(
     {lon: [-10, 10), lat: [0, 20)},
     ['temp_c']
 )
-AS OF SYSTEM TIME (extract(epoch from now()) * 1000 - 86400000);
+AS OF SYSTEM TIME '2026-06-06T00:00:00Z';
 ```
 
 ### Lineage and Compliance
@@ -153,12 +156,14 @@ UPDATE transactions SET status = 'CONFIRMED' WHERE id = 'txn-1';
 UPDATE transactions SET status = 'SETTLED' WHERE id = 'txn-1';
 
 -- Audit: show full timeline of this transaction
-SELECT status, system_time FROM transactions
+SELECT status, _ts_system FROM transactions
 WHERE id = 'txn-1'
-AS OF SYSTEM TIME NULL  -- special: returns all versions in system time order
-ORDER BY system_time ASC;
+AS OF SYSTEM TIME NULL  -- returns every system-time version, ascending
+ORDER BY _ts_system ASC;
 ```
 
+`AS OF SYSTEM TIME NULL` returns every system-time version of each matching row, ordered ascending, with the `_ts_system` column projected into the result. It is supported on the Document (strict and schemaless), Columnar, and Timeseries engines. It is not supported on the Graph or Array engines, nor when reading through a database clone — those return a typed error rather than collapsing to a single version.
+
 ## Index Engines and Temporal Composition
 
 Vector, FTS, KV, and Spatial do not carry temporal columns. This is not an oversight — it is the correct architectural decomposition. These four are **index engines**: they point at records that live in data-bearing engines. Indexes don't have time; the records they point to do.
@@ -185,7 +190,7 @@ CREATE VECTOR INDEX idx_product_vec ON product_embeddings METRIC cosine DIM 384;
 SELECT p.product_id, p.description,
        vector_distance(p.embedding, $query_vec) AS score
 FROM product_embeddings p
-AS OF SYSTEM TIME (extract(epoch from now()) * 1000 - 2592000000)
+AS OF SYSTEM TIME '2026-05-08T00:00:00Z'
 WHERE p.id IN (
     SEARCH product_embeddings USING VECTOR(embedding, $query_vec, 20)
 )
@@ -212,7 +217,7 @@ CREATE SEARCH INDEX ON articles FIELDS title, body ANALYZER 'english';
 -- 3. Bitemporal FTS: search the index state as it existed yesterday
 SELECT id, title
 FROM articles
-AS OF SYSTEM TIME (extract(epoch from now()) * 1000 - 86400000)
+AS OF SYSTEM TIME '2026-06-06T00:00:00Z'
 WHERE text_match(body, 'distributed consensus raft')
 ORDER BY bm25_score(body, 'distributed consensus raft') DESC
 LIMIT 20;
@@ -275,7 +280,7 @@ SELECT value FROM config_entries WHERE key = 'feature_x_enabled';
 
 -- Audit: what was the value 7 days ago?
 SELECT value FROM config_entries
-AS OF SYSTEM TIME (extract(epoch from now()) * 1000 - 604800000)
+AS OF SYSTEM TIME '2026-05-31T00:00:00Z'
 WHERE key = 'feature_x_enabled';
 ```
 

docs/bitemporal.md

@@ -166,7 +166,7 @@ Columnar collections track both system time and valid time, enabling corrections
 SELECT level, COUNT(*) FROM logs
 WHERE ts > now() - INTERVAL '1 hour'
 GROUP BY level
-AS OF SYSTEM TIME (extract(epoch from now()) * 1000 - 3600000);
+AS OF SYSTEM TIME '2026-06-07T12:00:00Z';
 
 -- Query records valid at a specific date (forecast corrections, backdated data)
 SELECT host, cpu_usage FROM metrics

docs/columnar.md

@@ -167,7 +167,7 @@ Both schemaless and strict documents support bitemporal queries — tracking sys
 ```sql
 -- Query documents as they existed yesterday (system time)
 SELECT * FROM users
-AS OF SYSTEM TIME (extract(epoch from now()) * 1000 - 86400000);
+AS OF SYSTEM TIME '2026-06-06T00:00:00Z';
 
 -- Query documents that were valid at a past date (valid time)
 SELECT * FROM users

docs/documents.md

@@ -39,13 +39,20 @@ CREATE SEARCH INDEX ON articles FIELDS title, body
     ANALYZER 'english'
     FUZZY true;
 
--- Basic search
+-- Basic search with text_match
 SELECT title, bm25_score(body, 'distributed database rust') AS score
 FROM articles
 WHERE text_match(body, 'distributed database rust')
 ORDER BY score DESC
 LIMIT 20;
 
+-- Using SEARCH() as an alias (equivalent to text_match)
+SELECT title, bm25_score(body, 'distributed database rust') AS score
+FROM articles
+WHERE SEARCH(body, 'distributed database rust')
+ORDER BY score DESC
+LIMIT 20;
+
 -- Fuzzy search (handles typos)
 SELECT title FROM articles
 WHERE text_match(title, 'databse', { fuzzy: true, distance: 2 });