feat(analytics): timezone-aware date bucketing (ADR-0053 Phase 2) (#1982)

.changeset/tz-aware-analytics-bucketing.md

@@ -0,0 +1,16 @@
+---
+"@objectstack/core": minor
+"@objectstack/spec": minor
+"@objectstack/objectql": minor
+"@objectstack/service-analytics": minor
+---
+
+feat(analytics): timezone-aware date bucketing (ADR-0053 Phase 2)
+
+Analytics day/week/month/quarter/year buckets now resolve on a **reference timezone's** calendar days, so a row near a tz day-boundary lands in the bucket a user in that zone would expect — identically on SQLite and Postgres.
+
+Per ADR-0053 decision **D2**, bucketing is done **in-memory, uniformly** for non-UTC zones rather than emitting dialect-specific `date_trunc … AT TIME ZONE` (SQLite has no tz database and MySQL needs tz tables loaded, so splitting by dialect would shift bucket boundaries for the same data). `engine.aggregate({ timezone })` therefore forces the in-memory aggregation path when a non-UTC reference tz is set — the date-range `where` still goes to the driver, so only matching rows are fetched. **UTC / unset keeps the native driver fast path unchanged.**
+
+- New shared `calendarPartsInTz` / `calendarPartsInTzOrUtc` util in `@objectstack/core` (DST-safe via `Intl.DateTimeFormat`, never hand-rolled offset math; falls back to UTC for an unset/`'UTC'`/invalid zone).
+- `EngineAggregateOptions` and the analytics `executeAggregate` bridge / `ObjectQLStrategy` thread the reference timezone (sourced from the dataset selection / `ExecutionContext`) through to `applyInMemoryAggregation` → `bucketDateValue`, and the draft-preview evaluator's `bucketDate`.
+- `formatDateBucket` (dimension labels) stays UTC-only by design: it re-labels values that were *already* bucketed upstream, so re-applying a timezone there would shift a correct bucket by a day.

packages/core/src/index.ts

@@ -23,6 +23,9 @@ export * from './security/index.js';
 // Export environment utilities
 export * from './utils/env.js';
 
+// Export timezone-aware calendar utilities (ADR-0053 Phase 2)
+export * from './utils/datetime.js';
+
 // Export in-memory fallbacks for core-criticality services
 export * from './fallbacks/index.js';
 

packages/core/src/utils/datetime.ts

@@ -0,0 +1,60 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * Timezone-aware calendar utilities (ADR-0053 Phase 2).
+ *
+ * The one primitive everything else builds on is {@link calendarPartsInTz}:
+ * the year/month/day an instant falls on *as seen in a reference timezone*.
+ * It uses `Intl.DateTimeFormat().formatToParts()` so DST transitions are
+ * handled by the platform's tz database — never hand-rolled offset math, which
+ * is the classic source of off-by-one-hour bucket errors.
+ *
+ * This lives in `@objectstack/core` (not `@objectstack/formula`) because both
+ * the ObjectQL aggregation engine and the analytics service need it and both
+ * already depend on core, whereas neither depends on formula's public surface.
+ * (`@objectstack/formula` keeps its own private copy for `today()`/`daysFromNow`
+ * to avoid a layering dependency on core.)
+ */
+
+/** Calendar-day parts in a reference timezone. `month` is 1-12. */
+export interface CalendarParts {
+  year: number;
+  month: number;
+  day: number;
+}
+
+/**
+ * The year/month/day an instant falls on in `tz`. Throws if `tz` is not a
+ * valid IANA zone (callers treat that as a fall-through to UTC).
+ */
+export function calendarPartsInTz(d: Date, tz: string): CalendarParts {
+  const parts = new Intl.DateTimeFormat('en-US', {
+    timeZone: tz,
+    year: 'numeric',
+    month: '2-digit',
+    day: '2-digit',
+  }).formatToParts(d);
+  const get = (t: string) => Number(parts.find((p) => p.type === t)?.value);
+  return { year: get('year'), month: get('month'), day: get('day') };
+}
+
+/**
+ * The calendar-day parts of an instant, in `tz` when it's a real non-UTC zone,
+ * otherwise in UTC. Never throws: an unset, `'UTC'`, or invalid zone falls back
+ * to the UTC calendar day. This is the safe entry point for bucketing code that
+ * must degrade to the historical UTC behavior rather than error.
+ */
+export function calendarPartsInTzOrUtc(d: Date, tz?: string): CalendarParts {
+  if (tz && tz !== 'UTC') {
+    try {
+      return calendarPartsInTz(d, tz);
+    } catch {
+      // unknown zone → fall through to UTC
+    }
+  }
+  return {
+    year: d.getUTCFullYear(),
+    month: d.getUTCMonth() + 1,
+    day: d.getUTCDate(),
+  };
+}

packages/objectql/src/engine-aggregate-timezone.test.ts

@@ -0,0 +1,102 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+//
+// ADR-0053 Phase 2 (D2): `engine.aggregate({ timezone })` routing.
+//
+// Native driver date bucketing (`date_trunc`) is UTC-only, so a non-UTC
+// reference timezone must force the in-memory path (uniform across drivers)
+// while UTC / unset keeps the native fast path. These tests pin both halves
+// using a driver that advertises native day bucketing and records whether its
+// native `aggregate()` was taken.
+
+import { describe, it, expect } from 'vitest';
+import { ObjectQL } from './engine.js';
+
+/**
+ * A driver advertising native `day` granularity. Its native `aggregate()`
+ * "buckets" in UTC (the only thing a real SQL `date_trunc` can do here) and
+ * flags that it ran, so the test can assert which path the engine chose.
+ */
+function makeBucketingDriver(rows: any[]) {
+  let nativeAggregateCalls = 0;
+  const driver: any = {
+    name: 'bucketing-mock',
+    version: '0.0.0',
+    supports: { queryDateGranularity: { day: true, week: true, month: true, quarter: true, year: true } },
+    async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; },
+    async find() { return rows.slice(); },
+    findStream() { throw new Error('ni'); },
+    async findOne() { return rows[0] ?? null; },
+    async create(_o: string, d: any) { return d; },
+    async update(_o: string, _id: string, d: any) { return d; },
+    async delete() { return true; },
+    async count() { return rows.length; },
+    async bulkCreate(_o: string, r: any[]) { return r; },
+    async bulkUpdate() { return []; }, async bulkDelete() {},
+    async beginTransaction() { return { __trx: true, commit: async () => {}, rollback: async () => {} }; },
+    async commit() {}, async rollback() {},
+    async aggregate(_object: string, ast: any) {
+      nativeAggregateCalls += 1;
+      // Simulate UTC `date_trunc` day bucketing + sum.
+      const buckets = new Map<string, number>();
+      const gran = ast.groupBy?.[0];
+      const field = typeof gran === 'string' ? gran : gran.field;
+      for (const r of rows) {
+        const day = new Date(String(r[field])).toISOString().slice(0, 10);
+        buckets.set(day, (buckets.get(day) ?? 0) + Number(r.amount));
+      }
+      return Array.from(buckets, ([closed_at, total]) => ({ closed_at, total }));
+    },
+  };
+  return { driver, nativeCalls: () => nativeAggregateCalls };
+}
+
+// Two events 4h apart straddling the NY midnight: same UTC day (03-01),
+// different NY days (02-29 / 03-01).
+const ROWS = [
+  { closed_at: '2024-03-01T03:00:00.000Z', amount: 10 }, // NY 02-29
+  { closed_at: '2024-03-01T07:00:00.000Z', amount: 5 },  // NY 03-01
+];
+
+async function makeEngine(rows: any[]) {
+  const { driver, nativeCalls } = makeBucketingDriver(rows);
+  const engine = new ObjectQL();
+  engine.registerDriver(driver, true);
+  await engine.init();
+  engine.registry.registerObject({
+    name: 'deal',
+    fields: { closed_at: { type: 'date' }, amount: { type: 'number' } },
+  } as any);
+  return { engine, nativeCalls };
+}
+
+const dayBucket = {
+  groupBy: [{ field: 'closed_at', dateGranularity: 'day' }],
+  aggregations: [{ function: 'sum', field: 'amount', alias: 'total' }],
+};
+
+describe('engine.aggregate timezone routing (ADR-0053 Phase 2)', () => {
+  it('UTC / unset takes the native driver fast path', async () => {
+    const { engine, nativeCalls } = await makeEngine(ROWS);
+    const utc = await engine.aggregate('deal', { ...dayBucket, timezone: 'UTC' } as any);
+    expect(nativeCalls()).toBe(1);
+    expect(utc).toEqual([{ closed_at: '2024-03-01', total: 15 }]);
+
+    const unset = await engine.aggregate('deal', { ...dayBucket } as any);
+    expect(nativeCalls()).toBe(2); // native again
+    expect(unset).toEqual([{ closed_at: '2024-03-01', total: 15 }]);
+  });
+
+  it('non-UTC timezone forces in-memory bucketing on the reference day', async () => {
+    const { engine, nativeCalls } = await makeEngine(ROWS);
+    const ny = (await engine.aggregate('deal', {
+      ...dayBucket,
+      timezone: 'America/New_York',
+    } as any)).sort((a: any, b: any) => String(a.closed_at).localeCompare(String(b.closed_at)));
+
+    expect(nativeCalls()).toBe(0); // native path skipped
+    expect(ny).toEqual([
+      { closed_at: '2024-02-29', total: 10 },
+      { closed_at: '2024-03-01', total: 5 },
+    ]);
+  });
+});

packages/objectql/src/engine.ts

@@ -2484,16 +2484,27 @@ export class ObjectQL implements IDataEngine {
             if (!g?.dateGranularity) return true; // plain {field} object is fine
             return granularityCaps?.[g.dateGranularity] === true;
         });
-        if (typeof drv.aggregate === 'function' && allStructuredSupported) {
+        // ADR-0053 Phase 2 (D2): native driver date bucketing (`date_trunc`) is
+        // UTC-only — SQLite has no tz database and MySQL needs tz tables loaded,
+        // so pushing tz-aware bucketing down splits boundaries per dialect. When
+        // a non-UTC reference timezone is in play we therefore force the
+        // in-memory path: the date-range `where` still goes to the driver (only
+        // matching rows are fetched), but bucketing runs uniformly in JS so a
+        // row near a tz day-boundary lands identically on every driver.
+        const tz = query.timezone;
+        const hasDateBucket = structuredItems.some((g: any) => !!g?.dateGranularity);
+        const tzRequiresInMemory = !!tz && tz !== 'UTC' && hasDateBucket;
+        if (typeof drv.aggregate === 'function' && allStructuredSupported && !tzRequiresInMemory) {
             return drv.aggregate(object, ast, this.buildDriverOptions(opCtx.context));
         }
         // In-memory fallback path: ask the driver for raw rows, then bucket +
         // aggregate here. This guarantees `groupBy` (incl. structured items
         // carrying `dateGranularity`) and `aggregations` always work even on
         // drivers that have no native aggregation support (driver-rest,
-        // driver-memory, partial SQL drivers).
+        // driver-memory, partial SQL drivers), and is the path that honours a
+        // non-UTC reference timezone.
         const raw = await driver.find(object, ast, this.buildDriverOptions(opCtx.context));
-        return applyInMemoryAggregation(raw, ast);
+        return applyInMemoryAggregation(raw, ast, tz);
       });
 
       return opCtx.result as any[];

packages/objectql/src/in-memory-aggregation.test.ts

@@ -105,4 +105,56 @@ describe('bucketDateValue', () => {
     expect(bucketDateValue(null, 'month')).toBe('(null)');
     expect(bucketDateValue('not-a-date', 'month')).toBe('(null)');
   });
+
+  // ADR-0053 Phase 2 (D2): a non-UTC reference timezone shifts the calendar day.
+  describe('timezone-aware bucketing', () => {
+    // 2024-03-01T03:00Z is still 2024-02-29 (22:00) in America/New_York.
+    const nearMidnight = '2024-03-01T03:00:00.000Z';
+
+    it('buckets on the reference zone calendar day (day/month/quarter)', () => {
+      expect(bucketDateValue(nearMidnight, 'day', 'America/New_York')).toBe('2024-02-29');
+      expect(bucketDateValue(nearMidnight, 'month', 'America/New_York')).toBe('2024-02');
+      expect(bucketDateValue(nearMidnight, 'quarter', 'America/New_York')).toBe('2024-Q1');
+      // ...while UTC sees the next day/month.
+      expect(bucketDateValue(nearMidnight, 'day', 'UTC')).toBe('2024-03-01');
+      expect(bucketDateValue(nearMidnight, 'month', 'UTC')).toBe('2024-03');
+    });
+
+    it('shifts the ISO week when the zone moves the day across a Monday', () => {
+      // 2024-03-04T02:00Z is a Monday in UTC (ISO week 10) but still
+      // 2024-03-03 Sunday (ISO week 9) in America/New_York.
+      const mondayUtc = '2024-03-04T02:00:00.000Z';
+      expect(bucketDateValue(mondayUtc, 'week', 'UTC')).toBe('2024-W10');
+      expect(bucketDateValue(mondayUtc, 'week', 'America/New_York')).toBe('2024-W09');
+    });
+
+    it('falls back to UTC for unset / UTC / invalid zones', () => {
+      expect(bucketDateValue(nearMidnight, 'day')).toBe('2024-03-01');
+      expect(bucketDateValue(nearMidnight, 'day', 'UTC')).toBe('2024-03-01');
+      expect(bucketDateValue(nearMidnight, 'day', 'Not/AZone')).toBe('2024-03-01');
+    });
+
+    it('groups rows into the right tz bucket via applyInMemoryAggregation', () => {
+      // Two events 4h apart that straddle the NY midnight: in UTC they share
+      // the 2024-03-01 day; in NY they split across 02-29 and 03-01.
+      const rows = [
+        { closed_at: '2024-03-01T03:00:00.000Z', amount: 10 }, // NY: 02-29
+        { closed_at: '2024-03-01T07:00:00.000Z', amount: 5 },  // NY: 03-01
+      ];
+      const ast = {
+        groupBy: [{ field: 'closed_at', dateGranularity: 'day' as const }],
+        aggregations: [{ function: 'sum', field: 'amount', alias: 'total' }],
+      };
+      const utc = applyInMemoryAggregation(rows, ast, 'UTC');
+      expect(utc).toEqual([{ closed_at: '2024-03-01', total: 15 }]);
+
+      const ny = applyInMemoryAggregation(rows, ast, 'America/New_York').sort(
+        (a, b) => String(a.closed_at).localeCompare(String(b.closed_at)),
+      );
+      expect(ny).toEqual([
+        { closed_at: '2024-02-29', total: 10 },
+        { closed_at: '2024-03-01', total: 5 },
+      ]);
+    });
+  });
 });

packages/objectql/src/in-memory-aggregation.ts

@@ -25,15 +25,22 @@
 // invalid values bucket as the literal string `'(null)'` to remain
 // consistent with the client `useReportData` hook.
 
+import { calendarPartsInTzOrUtc } from '@objectstack/core';
 import type { QueryAST, GroupByNode, AggregationNode, DateGranularityValue } from '@objectstack/spec/data';
 
 /**
  * Group + aggregate raw rows according to the AST's `groupBy` /
  * `aggregations`. When neither is present, returns the rows unchanged.
+ *
+ * `timezone` (ADR-0053 Phase 2) shifts date bucketing to a reference timezone
+ * so a row near a tz day-boundary lands in the right day/week/month/quarter.
+ * It is only consulted by `groupBy` items carrying a `dateGranularity`; an
+ * unset or `'UTC'` value keeps the historical UTC bucketing.
  */
 export function applyInMemoryAggregation(
   rows: any[],
   ast: Pick<QueryAST, 'groupBy' | 'aggregations'>,
+  timezone?: string,
 ): any[] {
   const groupBy = (ast.groupBy ?? []) as GroupByNode[];
   const aggregations = (ast.aggregations ?? []) as AggregationNode[];
@@ -50,7 +57,7 @@ export function applyInMemoryAggregation(
     const parts: string[] = [];
     for (const g of groupBy) {
       const fieldName = typeof g === 'string' ? g : (g.alias ?? g.field);
-      const value = projectGroupValue(row, g);
+      const value = projectGroupValue(row, g, timezone);
       key[fieldName] = value;
       parts.push(`${fieldName}=${value}`);
     }
@@ -71,11 +78,11 @@ export function applyInMemoryAggregation(
   return out;
 }
 
-function projectGroupValue(row: any, g: GroupByNode): string {
+function projectGroupValue(row: any, g: GroupByNode, timezone?: string): string {
   const field = typeof g === 'string' ? g : g.field;
   const v = row?.[field];
   if (typeof g !== 'string' && g.dateGranularity) {
-    return bucketDateValue(v, g.dateGranularity);
+    return bucketDateValue(v, g.dateGranularity, timezone);
   }
   return v == null ? '(null)' : String(v);
 }
@@ -161,13 +168,23 @@ function toNumber(v: any): number {
 /**
  * Bucket a date-like value into an ISO-formatted period label. Weeks start
  * Monday and use ISO week numbering.
+ *
+ * `timezone` (ADR-0053 Phase 2) resolves the calendar day in a reference zone
+ * so an instant near a tz day-boundary buckets where a user in that zone would
+ * expect. An unset / `'UTC'` / invalid zone keeps the historical UTC bucketing.
+ * The y/m/d are taken in the reference zone and the ISO-week math then runs on
+ * a UTC date built from those parts — the parts already carry the zone shift,
+ * so the week boundary lands correctly without re-applying any offset.
  */
-export function bucketDateValue(value: unknown, granularity: DateGranularityValue): string {
+export function bucketDateValue(
+  value: unknown,
+  granularity: DateGranularityValue,
+  timezone?: string,
+): string {
   if (value == null) return '(null)';
   const d = value instanceof Date ? value : new Date(String(value));
   if (Number.isNaN(d.getTime())) return '(null)';
-  const y = d.getUTCFullYear();
-  const m = d.getUTCMonth() + 1;
+  const { year: y, month: m, day } = calendarPartsInTzOrUtc(d, timezone);
   switch (granularity) {
     case 'year':
       return String(y);
@@ -176,10 +193,10 @@ export function bucketDateValue(value: unknown, granularity: DateGranularityValu
     case 'month':
       return `${y}-${String(m).padStart(2, '0')}`;
     case 'day':
-      return `${y}-${String(m).padStart(2, '0')}-${String(d.getUTCDate()).padStart(2, '0')}`;
+      return `${y}-${String(m).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
     case 'week': {
       // ISO-8601 week date: week 1 contains the first Thursday of the year.
-      const target = new Date(Date.UTC(y, d.getUTCMonth(), d.getUTCDate()));
+      const target = new Date(Date.UTC(y, m - 1, day));
       const dayNum = (target.getUTCDay() + 6) % 7; // Mon=0..Sun=6
       target.setUTCDate(target.getUTCDate() - dayNum + 3);
       const firstThursday = new Date(Date.UTC(target.getUTCFullYear(), 0, 4));

packages/services/service-analytics/src/__tests__/preview-evaluator.test.ts

@@ -150,4 +150,14 @@ describe('evaluateAnalyticsQueryOverRows', () => {
     expect(matchesWhere({ a: 5 }, { a: { $in: [1, 5] } })).toBe(true);
     expect(matchesWhere({ a: 'Hello World' }, { a: { $contains: 'world' } })).toBe(true);
   });
+
+  it('helpers: bucketDate resolves the calendar day in a reference timezone', () => {
+    // 2024-03-01T03:00Z is still 2024-02-29 in America/New_York.
+    const near = '2024-03-01T03:00:00.000Z';
+    expect(bucketDate(near, 'day', 'America/New_York')).toBe('2024-02-29');
+    expect(bucketDate(near, 'month', 'America/New_York')).toBe('2024-02');
+    // Unset / UTC keep the historical UTC bucketing.
+    expect(bucketDate(near, 'day')).toBe('2024-03-01');
+    expect(bucketDate(near, 'day', 'UTC')).toBe('2024-03-01');
+  });
 });

packages/services/service-analytics/src/dimension-labels.ts

@@ -60,6 +60,12 @@ const pad = (n: number) => String(n).padStart(2, '0');
  *   month   → "2026-04"
  *   week    → "2026-04-13" (ISO date of the bucket)
  *   day     → "2026-04-15"
+ *
+ * Intentionally UTC-only (ADR-0053 Phase 2): timezone bucketing happens
+ * upstream in `bucketDate` / `bucketDateValue`, so by the time a value reaches
+ * here it is *already* the reference-zone bucket (often a label string like
+ * "2026-Q2"). Re-applying a timezone here would shift an already-correct
+ * `YYYY-MM-DD` day bucket by a day — this is a pure, idempotent re-labeler.
  */
 export function formatDateBucket(value: unknown, granularity?: DateGranularity | string): unknown {
   if (value == null || value instanceof Date === false) {