Merge pull request #1779 from objectstack-ai/feat/matrix-server-side-totals

feat(analytics): server-side totals for matrix reports (subtotals + grand total)

Author: os-zhuang

.changeset/matrix-server-side-totals.md

@@ -0,0 +1,6 @@
+---
+"@objectstack/spec": minor
+"@objectstack/service-analytics": minor
+---
+
+Server-side totals for matrix reports (#1753). `queryDataset` selections accept `totals: { groupings: string[][] }` — each grouping a subset of `selection.dimensions` to additionally aggregate by (`[]` = grand total); the marginal rows come back on `AnalyticsResult.totals` in request order. Each subtotal/grand total re-runs the full executor pipeline (measure-scoped filters, derived measures, compareTo) grouped only by that subset, so totals use each measure's true aggregate over the underlying rows — an `avg` total is the average of all rows, never an average of bucket averages (the ADR-0021 line that forbids client-side re-aggregation). Dimension display labels resolve on totals rows the same as the primary grid. A matrix report renderer asks for `{ groupings: [rowDims, columnDims, []] }` and renders the supplied totals row/column.

packages/services/service-analytics/src/__tests__/dataset-executor.test.ts

@@ -110,6 +110,78 @@ describe('DatasetExecutor', () => {
     expect(res.rows[0].win_rate).toBe(0.6);
   });
 
+  it('totals groupings re-run the selection per dimension subset (#1753)', async () => {
+    const matrix = DatasetSchema.parse({
+      name: 'hours', label: 'Hours', object: 'task',
+      dimensions: [
+        { name: 'owner', field: 'owner', type: 'string' },
+        { name: 'status', field: 'status', type: 'string' },
+      ],
+      measures: [{ name: 'avg_hours', aggregate: 'avg', field: 'est_hours' }],
+    });
+    const seen: AnalyticsQuery[] = [];
+    const svc = fakeService((q) => {
+      seen.push(q);
+      const dims = (q.dimensions ?? []).join(',');
+      if (dims === 'owner,status') return { rows: [{ owner: 'alice', status: 'done', avg_hours: 2 }], fields: [] };
+      if (dims === 'owner') return { rows: [{ owner: 'alice', avg_hours: 3 }], fields: [] };
+      if (dims === 'status') return { rows: [{ status: 'done', avg_hours: 4 }], fields: [] };
+      // grand total — the TRUE avg over all rows, not an avg of bucket avgs
+      return { rows: [{ avg_hours: 3.5 }], fields: [] };
+    });
+    const compiled = compileDataset(matrix);
+    const res = await new DatasetExecutor(svc).execute(compiled, {
+      dimensions: ['owner', 'status'], measures: ['avg_hours'],
+      order: { avg_hours: 'desc' }, limit: 50,
+      totals: { groupings: [['owner'], ['status'], []] },
+    });
+    expect(res.rows).toEqual([{ owner: 'alice', status: 'done', avg_hours: 2 }]);
+    expect(res.totals).toEqual([
+      { dimensions: ['owner'], rows: [{ owner: 'alice', avg_hours: 3 }] },
+      { dimensions: ['status'], rows: [{ status: 'done', avg_hours: 4 }] },
+      { dimensions: [], rows: [{ avg_hours: 3.5 }] },
+    ]);
+    // primary query keeps order/limit; totals queries drop them (totals cover
+    // the full selection, and an order key may reference a dropped dimension)
+    const primary = seen.find((q) => (q.dimensions ?? []).length === 2)!;
+    expect(primary.order).toEqual({ avg_hours: 'desc' });
+    expect(primary.limit).toBe(50);
+    for (const q of seen.filter((s) => (s.dimensions ?? []).length < 2)) {
+      expect(q.order).toBeUndefined();
+      expect(q.limit).toBeUndefined();
+    }
+  });
+
+  it('totals carry measure-scoped filters and derived measures', async () => {
+    const svc = fakeService((q) => {
+      const grandTotal = (q.dimensions ?? []).length === 0;
+      if (q.measures.includes('won_amount')) {
+        // measure-scoped filter must also reach the totals query
+        expect(JSON.stringify(q.where)).toContain('"stage":"won"');
+        return { rows: [grandTotal ? { won_amount: 120 } : { region: 'NA', won_amount: 60 }], fields: [] };
+      }
+      return { rows: [grandTotal ? { revenue: 200 } : { region: 'NA', revenue: 100 }], fields: [] };
+    });
+    const compiled = compileDataset(dataset);
+    const res = await new DatasetExecutor(svc).execute(compiled, {
+      dimensions: ['region'], measures: ['win_rate'],
+      totals: { groupings: [[]] },
+    });
+    expect(res.rows[0].win_rate).toBe(0.6);
+    expect(res.totals).toEqual([{ dimensions: [], rows: [{ revenue: 200, won_amount: 120, win_rate: 0.6 }] }]);
+  });
+
+  it('rejects a totals grouping that is not a subset of the selected dimensions', async () => {
+    const svc = fakeService(() => ({ rows: [], fields: [] }));
+    const compiled = compileDataset(dataset);
+    await expect(
+      new DatasetExecutor(svc).execute(compiled, {
+        dimensions: ['region'], measures: ['revenue'],
+        totals: { groupings: [['stage']] },
+      }),
+    ).rejects.toThrow(/not a subset of the selected dimensions/);
+  });
+
   it('compareTo runs a shifted query and attaches <measure>__compare', async () => {
     const seen: AnalyticsQuery[] = [];
     const svc = fakeService((q) => {

packages/services/service-analytics/src/__tests__/dimension-labels.test.ts

@@ -221,6 +221,21 @@ describe('AnalyticsService.queryDataset — label resolution (integration)', ()
     ]);
   });
 
+  it('resolves labels on server-side totals rows (#1753)', async () => {
+    const res = await service().queryDataset(dataset, {
+      dimensions: ['status', 'account'],
+      measures: ['task_count'],
+      totals: { groupings: [['account']] },
+    });
+    expect(res.totals).toEqual([{
+      dimensions: ['account'],
+      rows: [
+        { account: 'Acme Corp', task_count: 4 },
+        { account: 'Globex', task_count: 2 },
+      ],
+    }]);
+  });
+
   it('enriches measure fields with their display label + format', async () => {
     const labelledDataset = DatasetSchema.parse({
       name: 'sales_metrics',

packages/services/service-analytics/src/analytics-service.ts

@@ -404,6 +404,14 @@ export class AnalyticsService implements IAnalyticsService {
       if (dims.length) {
         try {
           await resolveDimensionLabels(dataset.object, dims, result.rows, this.labelResolver);
+          // Totals rows (#1753) carry dimension values too (a row subtotal is
+          // keyed by its row bucket) — resolve each grouping's own subset.
+          for (const total of result.totals ?? []) {
+            const subset = dims.filter((d) => total.dimensions.includes(d.name));
+            if (subset.length) {
+              await resolveDimensionLabels(dataset.object, subset, total.rows, this.labelResolver);
+            }
+          }
         } catch (e) {
           this.logger?.warn?.(`[Analytics] dimension label resolution failed for "${dataset.name}": ${String((e as Error)?.message ?? e)}`);
         }

packages/services/service-analytics/src/dataset-executor.ts

@@ -26,7 +26,10 @@ export type CompareTo = DatasetCompareTo;
  *   - applies measure-scoped filters via supplementary grouped queries,
  *   - evaluates derived measures (ratio/sum/difference/product) row-by-row (Q1),
  *   - shifts the query for `compareTo` (previousPeriod / previousYear) and
- *     attaches `<measure>__compare` columns.
+ *     attaches `<measure>__compare` columns,
+ *   - computes server-side totals (`selection.totals.groupings`, #1753) by
+ *     re-running the selection per dimension subset, so matrix subtotals and
+ *     the grand total use each measure's true aggregate.
  *
  * RLS/tenant scoping is NOT handled here — it is enforced inside the strategy
  * via the StrategyContext read-scope hook (D-C). This layer is pure query
@@ -136,6 +139,47 @@ export class DatasetExecutor {
     compiled: CompiledDataset,
     selection: DatasetSelection,
     context?: ExecutionContext,
+  ): Promise<AnalyticsResult> {
+    const result = await this.executeSelection(compiled, selection, context);
+
+    // Server-side totals (#1753) — re-run the selection grouped by each
+    // requested dimension subset, so a subtotal/grand total is the measure's
+    // TRUE aggregate over the underlying rows (an avg total is the average of
+    // all rows, not of bucket averages). Re-running the full pipeline keeps
+    // measure-scoped filters, derived measures, and compareTo consistent with
+    // the primary grid. order/limit/offset are dropped: totals cover the whole
+    // selection, and an order key may reference a dimension the grouping drops.
+    const groupings = selection.totals?.groupings;
+    if (groupings?.length) {
+      const selected = new Set(selection.dimensions ?? []);
+      const totals: NonNullable<AnalyticsResult['totals']> = [];
+      for (const grouping of groupings) {
+        const unknown = grouping.filter((d) => !selected.has(d));
+        if (unknown.length) {
+          throw new Error(
+            `[dataset-executor] totals grouping [${grouping.join(', ')}] is not a subset of the selected dimensions — unknown: ${unknown.join(', ')}.`,
+          );
+        }
+        const sub = await this.executeSelection(compiled, {
+          ...selection,
+          dimensions: grouping,
+          totals: undefined,
+          order: undefined,
+          limit: undefined,
+          offset: undefined,
+        }, context);
+        totals.push({ dimensions: grouping, rows: sub.rows });
+      }
+      result.totals = totals;
+    }
+
+    return result;
+  }
+
+  private async executeSelection(
+    compiled: CompiledDataset,
+    selection: DatasetSelection,
+    context?: ExecutionContext,
   ): Promise<AnalyticsResult> {
     const derivedByName = new Map(compiled.derived.map((d) => [d.name, d]));
     const selectedDerived = selection.measures

packages/spec/src/contracts/analytics-service.ts

@@ -75,6 +75,19 @@ export interface AnalyticsResult {
     }>;
     /** Generated SQL (if available) */
     sql?: string;
+    /**
+     * Marginal aggregates — one entry per `DatasetSelection.totals` grouping,
+     * in request order. Each entry's rows carry the grouping's dimension
+     * columns plus the same measure columns as `rows`, computed with the
+     * measure's true aggregate over the underlying data (never re-derived
+     * from bucketed values). The grand-total grouping (`[]`) yields a single
+     * dimensionless row.
+     */
+    totals?: Array<{
+        /** The dimension subset this marginal was grouped by ([] = grand total). */
+        dimensions: string[];
+        rows: Record<string, unknown>[];
+    }>;
 }
 
 /**
@@ -121,6 +134,19 @@ export interface DatasetSelection {
     offset?: number;
     /** Compare-to directive — runs a shifted query and attaches `<measure>__compare`. */
     compareTo?: DatasetCompareTo;
+    /**
+     * Server-side totals (matrix subtotals + grand total). Each grouping is a
+     * subset of `dimensions` to additionally aggregate by; the selection is
+     * re-run grouped only by those dimensions, so every total is the measure's
+     * TRUE aggregate over the underlying rows — an `avg` total is the average
+     * over all rows, not an average of bucket averages (the ADR-0021
+     * governance line that forbids client-side re-aggregation). `[]` requests
+     * the grand total. A matrix report asks for
+     * `{ groupings: [rowDims, columnDims, []] }`. Results arrive on
+     * `AnalyticsResult.totals` in request order. `order`/`limit`/`offset` do
+     * not apply to totals queries — totals always cover the full selection.
+     */
+    totals?: { groupings: string[][] };
     timezone?: string;
 }