fix(driver-sql): coerce ISO/Date filter values for Field.datetime columns

os-zhuang · Copilot · os-zhuang · commit d48a0d67240b · 2026-05-25T10:29:34.000+08:00
Field.datetime() values stored via better-sqlite3 land as INTEGER
milliseconds, but filter macros and most user input arrive as ISO
date strings ('2026-01-01') or JS Date objects. SQLite's type
affinity rules turn TEXT-vs-INTEGER comparisons into numeric
compares of leading digits, so

  WHERE published_at &gt;= '2026-01-01'   -- 2026 &lt; 1778… → no rows

silently matched nothing. Charts that bypass filters happened to
work, masking the issue everywhere except KPI / metric widgets.

This change tracks date/datetime columns per table during
initObjects() and adds coerceFilterValue() to normalise the
right-hand side of every operator inside applyFilters() and
applyFilterCondition():

- Field.datetime → INTEGER milliseconds (Date.getTime(), parsed ISO,
  numeric strings, or already-numeric inputs all converge).
- Field.date → YYYY-MM-DD TEXT (so JS Date objects don't leak as
  full ISO timestamps and break lexicographic comparison).

Adds sql-driver-datetime-filter.test.ts covering $gte/$lt range,
full ISO timestamps, JS Date and numeric epoch inputs across both
column types. All 118 existing driver-sql tests still pass.

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;
diff --git a/packages/plugins/driver-sql/src/sql-driver-datetime-filter.test.ts b/packages/plugins/driver-sql/src/sql-driver-datetime-filter.test.ts
@@ -0,0 +1,113 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * Regression: filters on `Field.datetime()` columns must work even when
+ * the comparand is an ISO date string. The platform stores datetime
+ * values as INTEGER milliseconds (via better-sqlite3's Date binding),
+ * so `where: { published_at: { $gte: '2026-01-01' } }` used to compile
+ * to a TEXT-vs-INTEGER affinity compare that never matched.
+ *
+ * This suite covers ISO string, full ISO timestamp, JS Date and numeric
+ * inputs for both `Field.datetime()` and `Field.date()` columns.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { SqlDriver } from '../src/index.js';
+
+describe('SqlDriver datetime filter coercion', () => {
+  let driver: SqlDriver;
+
+  beforeEach(async () => {
+    driver = new SqlDriver({
+      client: 'better-sqlite3',
+      connection: { filename: ':memory:' },
+      useNullAsDefault: true,
+    });
+
+    await driver.initObjects([
+      {
+        name: 'publication',
+        fields: {
+          title: { type: 'string' },
+          published_at: { type: 'datetime' },
+          period_start: { type: 'date' },
+          views: { type: 'integer' },
+        },
+      },
+    ]);
+
+    // Insert with real Date objects so better-sqlite3 stores them as
+    // INTEGER milliseconds — the exact path the seed loader takes via
+    // `cel\`daysAgo(N)\``.
+    await driver.create('publication', {
+      id: 'p1', title: 'Old', views: 100,
+      published_at: new Date('2025-01-15T00:00:00Z'),
+      period_start: '2025-01-15',
+    }, { bypassTenantAudit: true });
+    await driver.create('publication', {
+      id: 'p2', title: 'New', views: 200,
+      published_at: new Date('2026-03-20T12:00:00Z'),
+      period_start: '2026-03-20',
+    }, { bypassTenantAudit: true });
+    await driver.create('publication', {
+      id: 'p3', title: 'Newer', views: 300,
+      published_at: new Date('2026-05-25T08:00:00Z'),
+      period_start: '2026-05-25',
+    }, { bypassTenantAudit: true });
+  });
+
+  afterEach(async () => {
+    await driver.disconnect();
+  });
+
+  it('matches datetime $gte against an ISO date string', async () => {
+    const rows = await driver.find('publication', {
+      where: { published_at: { $gte: '2026-01-01' } },
+      orderBy: [{ field: 'published_at', order: 'asc' }],
+    });
+    expect(rows.map((r: any) => r.id)).toEqual(['p2', 'p3']);
+  });
+
+  it('matches datetime range with $gte / $lt', async () => {
+    const rows = await driver.find('publication', {
+      where: { published_at: { $gte: '2026-01-01', $lt: '2026-05-01' } },
+    });
+    expect(rows.map((r: any) => r.id)).toEqual(['p2']);
+  });
+
+  it('accepts a full ISO timestamp', async () => {
+    const rows = await driver.find('publication', {
+      where: { published_at: { $gte: '2026-05-25T00:00:00.000Z' } },
+    });
+    expect(rows.map((r: any) => r.id)).toEqual(['p3']);
+  });
+
+  it('accepts a JS Date object', async () => {
+    const rows = await driver.find('publication', {
+      where: { published_at: { $gte: new Date('2026-01-01T00:00:00Z') } },
+    });
+    expect(rows.map((r: any) => r.id).sort()).toEqual(['p2', 'p3']);
+  });
+
+  it('accepts a numeric epoch millisecond value', async () => {
+    const ms = Date.parse('2026-01-01T00:00:00Z');
+    const rows = await driver.find('publication', {
+      where: { published_at: { $gte: ms } },
+    });
+    expect(rows.map((r: any) => r.id).sort()).toEqual(['p2', 'p3']);
+  });
+
+  it('still filters non-date columns normally', async () => {
+    const rows = await driver.find('publication', {
+      where: { views: { $gte: 200 } },
+    });
+    expect(rows.map((r: any) => r.id).sort()).toEqual(['p2', 'p3']);
+  });
+
+  it('matches date (YYYY-MM-DD) columns with ISO comparand', async () => {
+    const rows = await driver.find('publication', {
+      where: { period_start: { $gte: '2026-01-01' } },
+    });
+    expect(rows.map((r: any) => r.id).sort()).toEqual(['p2', 'p3']);
+  });
+});
diff --git a/packages/plugins/driver-sql/src/sql-driver.ts b/packages/plugins/driver-sql/src/sql-driver.ts
@@ -141,6 +141,8 @@ export class SqlDriver implements IDataDriver {
   protected config: Knex.Config;
   protected jsonFields: Record<string, string[]> = {};
   protected booleanFields: Record<string, string[]> = {};
+  protected dateFields: Record<string, Set<string>> = {};
+  protected datetimeFields: Record<string, Set<string>> = {};
   protected tablesWithTimestamps: Set<string> = new Set();
   /**
    * Autonumber field configs per table, captured during initObjects.
@@ -1019,6 +1021,12 @@ export class SqlDriver implements IDataDriver {
           if (type === 'boolean') {
             booleanCols.push(name);
           }
+          if (type === 'date') {
+            (this.dateFields[tableName] ??= new Set()).add(name);
+          }
+          if (type === 'datetime') {
+            (this.datetimeFields[tableName] ??= new Set()).add(name);
+          }
           if (type === 'auto_number' || type === 'autonumber') {
             const fmt = typeof field.format === 'string' && field.format
               ? field.format
@@ -1247,8 +1255,81 @@ export class SqlDriver implements IDataDriver {
 
   // ── Filter helpers ──────────────────────────────────────────────────────────
 
+  /**
+   * Resolve the underlying table name for a Knex query builder so we can
+   * look up column type metadata (date/datetime maps populated during
+   * `initObjects`). Returns null when the builder is not table-scoped yet.
+   */
+  protected tableNameForBuilder(builder: any): string | null {
+    const t = builder?._single?.table;
+    if (typeof t === 'string') return t;
+    return null;
+  }
+
+  /**
+   * Normalise a filter value for a single column so the comparison the
+   * driver sends to SQLite matches the on-disk representation.
+   *
+   * The platform stores `Field.datetime()` values as INTEGER milliseconds
+   * (the result of passing a JS `Date` through better-sqlite3) but date
+   * macros like `{last_quarter_start}` expand to an ISO `YYYY-MM-DD` string
+   * client-side. Without coercion the SQL becomes `published_at >= '2026-…'`
+   * which collapses to a TEXT-vs-INTEGER affinity compare and never
+   * matches. We translate the ISO/Date/numeric inputs into the storage
+   * type so the comparison works.
+   *
+   * For `Field.date()` we keep ISO TEXT but normalise Date objects to
+   * `YYYY-MM-DD` for the same reason.
+   */
+  protected coerceFilterValue(table: string | null, field: string, value: any): any {
+    if (value == null || !table) return value;
+    if (Array.isArray(value)) return value.map((v) => this.coerceFilterValue(table, field, v));
+
+    const isDatetime = this.datetimeFields[table]?.has(field);
+    const isDate = this.dateFields[table]?.has(field);
+    if (!isDatetime && !isDate) return value;
+
+    const toMs = (v: any): number | null => {
+      if (v instanceof Date) return v.getTime();
+      if (typeof v === 'number' && Number.isFinite(v)) return v;
+      if (typeof v === 'string') {
+        const trimmed = v.trim();
+        if (trimmed === '') return null;
+        if (/^-?\d+$/.test(trimmed)) {
+          const n = Number(trimmed);
+          if (Number.isFinite(n)) return n;
+        }
+        // Treat bare YYYY-MM-DD as start-of-day UTC; full ISO is parsed
+        // as-is so timezones round-trip correctly.
+        const iso = /^\d{4}-\d{2}-\d{2}$/.test(trimmed) ? `${trimmed}T00:00:00.000Z` : trimmed;
+        const n = Date.parse(iso);
+        return Number.isFinite(n) ? n : null;
+      }
+      return null;
+    };
+
+    if (isDatetime) {
+      const ms = toMs(value);
+      return ms == null ? value : ms;
+    }
+
+    // Field.date — normalise to YYYY-MM-DD.
+    if (value instanceof Date) {
+      const y = value.getUTCFullYear();
+      const m = String(value.getUTCMonth() + 1).padStart(2, '0');
+      const d = String(value.getUTCDate()).padStart(2, '0');
+      return `${y}-${m}-${d}`;
+    }
+    if (typeof value === 'string') {
+      const trimmed = value.trim();
+      if (/^\d{4}-\d{2}-\d{2}/.test(trimmed)) return trimmed.slice(0, 10);
+    }
+    return value;
+  }
+
   protected applyFilters(builder: Knex.QueryBuilder, filters: any) {
     if (!filters) return;
+    const table = this.tableNameForBuilder(builder);
 
     if (!Array.isArray(filters) && typeof filters === 'object') {
       const hasMongoOperators = Object.keys(filters).some(
@@ -1260,13 +1341,13 @@ export class SqlDriver implements IDataDriver {
       );
 
       if (hasMongoOperators) {
-        this.applyFilterCondition(builder, filters);
+        this.applyFilterCondition(builder, filters, 'and', table);
         return;
       }
 
       for (const [key, value] of Object.entries(filters)) {
         if (['limit', 'offset', 'fields', 'orderBy'].includes(key)) continue;
-        builder.where(key, value as any);
+        builder.where(key, this.coerceFilterValue(table, key, value) as any);
       }
       return;
     }
@@ -1288,6 +1369,7 @@ export class SqlDriver implements IDataDriver {
 
         if (isCriterion) {
           const field = this.mapSortField(fieldRaw);
+          const coerced = this.coerceFilterValue(table, field, value);
           const apply = (b: any) => {
             const method = nextJoin === 'or' ? 'orWhere' : 'where';
             const methodIn = nextJoin === 'or' ? 'orWhereIn' : 'whereIn';
@@ -1300,19 +1382,19 @@ export class SqlDriver implements IDataDriver {
 
             switch (op) {
               case '=':
-                b[method](field, value);
+                b[method](field, coerced);
                 break;
               case '!=':
-                b[method](field, '<>', value);
+                b[method](field, '<>', coerced);
                 break;
               case 'in':
-                b[methodIn](field, value);
+                b[methodIn](field, coerced);
                 break;
               case 'nin':
-                b[methodNotIn](field, value);
+                b[methodNotIn](field, coerced);
                 break;
               default:
-                b[method](field, op, value);
+                b[method](field, op, coerced);
             }
           };
           apply(builder);
@@ -1328,15 +1410,16 @@ export class SqlDriver implements IDataDriver {
     }
   }
 
-  protected applyFilterCondition(builder: Knex.QueryBuilder, condition: any, logicalOp: 'and' | 'or' = 'and') {
+  protected applyFilterCondition(builder: Knex.QueryBuilder, condition: any, logicalOp: 'and' | 'or' = 'and', tableHint?: string | null) {
     if (!condition || typeof condition !== 'object') return;
+    const table = tableHint ?? this.tableNameForBuilder(builder);
 
     for (const [key, value] of Object.entries(condition)) {
       if (key === '$and' && Array.isArray(value)) {
         builder.where((qb) => {
           for (const sub of value) {
             qb.where((subQb) => {
-              this.applyFilterCondition(subQb, sub, 'and');
+              this.applyFilterCondition(subQb, sub, 'and', table);
             });
           }
         });
@@ -1345,54 +1428,55 @@ export class SqlDriver implements IDataDriver {
         (builder as any)[method]((qb: any) => {
           for (const sub of value) {
             qb.orWhere((subQb: any) => {
-              this.applyFilterCondition(subQb, sub, 'or');
+              this.applyFilterCondition(subQb, sub, 'or', table);
             });
           }
         });
       } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
         const field = this.mapSortField(key);
         for (const [op, opValue] of Object.entries(value as Record<string, any>)) {
           const method = logicalOp === 'or' ? 'orWhere' : 'where';
+          const coerced = this.coerceFilterValue(table, field, opValue);
           switch (op) {
             case '$eq':
-              (builder as any)[method](field, opValue);
+              (builder as any)[method](field, coerced);
               break;
             case '$ne':
-              (builder as any)[method](field, '<>', opValue);
+              (builder as any)[method](field, '<>', coerced);
               break;
             case '$gt':
-              (builder as any)[method](field, '>', opValue);
+              (builder as any)[method](field, '>', coerced);
               break;
             case '$gte':
-              (builder as any)[method](field, '>=', opValue);
+              (builder as any)[method](field, '>=', coerced);
               break;
             case '$lt':
-              (builder as any)[method](field, '<', opValue);
+              (builder as any)[method](field, '<', coerced);
               break;
             case '$lte':
-              (builder as any)[method](field, '<=', opValue);
+              (builder as any)[method](field, '<=', coerced);
               break;
             case '$in': {
               const mIn = logicalOp === 'or' ? 'orWhereIn' : 'whereIn';
-              (builder as any)[mIn](field, opValue as any[]);
+              (builder as any)[mIn](field, coerced as any[]);
               break;
             }
             case '$nin': {
               const mNotIn = logicalOp === 'or' ? 'orWhereNotIn' : 'whereNotIn';
-              (builder as any)[mNotIn](field, opValue as any[]);
+              (builder as any)[mNotIn](field, coerced as any[]);
               break;
             }
             case '$contains':
               (builder as any)[method](field, 'like', `%${opValue}%`);
               break;
             default:
-              (builder as any)[method](field, opValue);
+              (builder as any)[method](field, coerced);
           }
         }
       } else {
         const field = this.mapSortField(key);
         const method = logicalOp === 'or' ? 'orWhere' : 'where';
-        (builder as any)[method](field, value as any);
+        (builder as any)[method](field, this.coerceFilterValue(table, field, value) as any);
       }
     }
   }
