diff --git a/.changeset/adr-0015-external-columnmap.md b/.changeset/adr-0015-external-columnmap.md new file mode 100644 index 0000000000..68258ef45c --- /dev/null +++ b/.changeset/adr-0015-external-columnmap.md @@ -0,0 +1,12 @@ +--- +"@objectstack/driver-sql": patch +--- + +feat(driver-sql): honor `external.columnMap` on federated (external) objects (ADR-0015). + +When a federated object declares `external.columnMap` ({ remoteColumn -> localField }), +the SQL driver now translates queries to the physical remote columns: WHERE and +ORDER BY map local fields to remote columns (value coercion stays keyed by the local +field), `formatOutput` renames remote-column keys back to local field names on read, +and write payloads are key-remapped. Managed objects and external objects without a +columnMap are unchanged (the resolver falls back to the existing per-site behavior). diff --git a/docs/adr/0015-external-datasource-federation.md b/docs/adr/0015-external-datasource-federation.md index edb3a6788e..f9f0e879d1 100644 --- a/docs/adr/0015-external-datasource-federation.md +++ b/docs/adr/0015-external-datasource-federation.md @@ -929,13 +929,21 @@ contract validated green and then queried the wrong (non-existent) table. is driven purely by the declared field types. Keep external object fields to well-understood scalar types. +### `external.columnMap` (added in a follow-up) + +`external.columnMap` ({ remoteColumn → localField }) is now honored on the read + +write paths. `registerExternalObject` inverts it to localField → remoteColumn; +`SqlDriver` translates WHERE + ORDER BY to the physical remote column (coercion +stays keyed by the local field), `formatOutput` renames remote-column keys back to +local field names, and write payloads are key-remapped after `formatInput`. +Managed objects and external objects without a columnMap are unchanged (the +resolver falls back to the existing per-site behavior). `field.columnName` +reconciliation (its inverse) is left untouched — `columnMap` is the +external-binding mechanism. + ### Explicitly out of scope (separate follow-ups) -1. **`external.columnMap`** (remote column name ≠ local field key). The driver's - `select` / `where` / `orderBy` do not currently apply column-name translation, - and `columnMap` is the inverse of per-field `field.columnName` — reconciling - the two into one source of truth is its own change. -2. **Native-analytics SQL over external objects** — the analytics service compiles +1. **Native-analytics SQL over external objects** — the analytics service compiles its own `FROM ""` outside the driver and needs the same remote-table awareness. 3. **Auto-connecting declared datasources** as queryable ObjectQL drivers in the diff --git a/packages/plugins/driver-sql/src/sql-driver-external-columnmap.test.ts b/packages/plugins/driver-sql/src/sql-driver-external-columnmap.test.ts new file mode 100644 index 0000000000..759ca5cb88 --- /dev/null +++ b/packages/plugins/driver-sql/src/sql-driver-external-columnmap.test.ts @@ -0,0 +1,128 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Federation columnMap tests (ADR-0015 §18). + * + * An external object can bind to a remote table whose COLUMN names differ from + * the local field names via `external.columnMap` ({ remoteColumn -> localField }). + * Reads must come back keyed by local field names (with coercion), and WHERE / + * ORDER BY must translate local fields to the physical remote columns. + */ + +import { describe, it, expect, afterAll } from 'vitest'; +import { rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { SqlDriver } from '../src/index.js'; + +// Remote table with deliberately non-matching column names. +const REMOTE = { + name: 'legacy_cust', + fields: { + cust_id: { type: 'text' }, + full_name: { type: 'text' }, + region_code: { type: 'text' }, + ltv: { type: 'number' }, + signup_dt: { type: 'date' }, + }, +}; + +// Local federated object: clean field names, bound to the remote columns. +const COLUMN_MAP = { full_name: 'name', region_code: 'region', ltv: 'value', signup_dt: 'signed_up', cust_id: 'id' }; +const LOCAL_FIELDS = { + name: { type: 'text' }, + region: { type: 'text' }, + value: { type: 'number' }, + signed_up: { type: 'date' }, +}; + +let file: string; +afterAll(() => { if (file) { try { rmSync(file, { force: true }); } catch { /* ignore */ } } }); + +async function seedRemote(path: string) { + const fx = new SqlDriver({ client: 'better-sqlite3', connection: { filename: path }, useNullAsDefault: true }) as any; + fx.name = 'fx'; + await fx.connect(); + await fx.initObjects([REMOTE]); + await fx.create('legacy_cust', { cust_id: 'c1', full_name: 'Aurora', region_code: 'NA', ltv: 480, signup_dt: '2026-01-10' }); + await fx.create('legacy_cust', { cust_id: 'c2', full_name: 'Borealis', region_code: 'EU', ltv: 312, signup_dt: '2026-02-15' }); + await fx.create('legacy_cust', { cust_id: 'c3', full_name: 'Cyan', region_code: 'EU', ltv: 95, signup_dt: '2026-03-01' }); + await fx.disconnect(); +} + +function ext(path: string): any { + const d = new SqlDriver({ client: 'better-sqlite3', connection: { filename: path }, useNullAsDefault: true, schemaMode: 'external' } as any) as any; + d.name = 'extds'; + return d; +} + +describe('SqlDriver external columnMap (ADR-0015 §18)', () => { + it('reads come back keyed by local field names from differently-named remote columns', async () => { + file = join(tmpdir(), `os-cm-${process.pid}-${Date.now()}.db`); + await seedRemote(file); + const d = ext(file); + await d.connect(); + try { + d.registerExternalObject({ name: 'cm_customer', external: { remoteName: 'legacy_cust', columnMap: COLUMN_MAP }, fields: LOCAL_FIELDS }); + + const rows = await d.find('cm_customer', {}); + expect(rows).toHaveLength(3); + const aurora = rows.find((r: any) => r.name === 'Aurora'); + expect(aurora).toBeTruthy(); + // Local field names present; remote column names absent. + expect(aurora.id).toBe('c1'); + expect(aurora.region).toBe('NA'); + expect(typeof aurora.value).toBe('number'); + expect(aurora.value).toBe(480); + expect(aurora.signed_up).toBe('2026-01-10'); + expect(aurora.full_name).toBeUndefined(); + expect(aurora.region_code).toBeUndefined(); + expect(aurora.ltv).toBeUndefined(); + } finally { await d.disconnect(); } + }); + + it('WHERE (object + array form) translates local field -> remote column', async () => { + const d = ext(file); + await d.connect(); + try { + d.registerExternalObject({ name: 'cm_customer', external: { remoteName: 'legacy_cust', columnMap: COLUMN_MAP }, fields: LOCAL_FIELDS }); + // object form: region -> region_code + const eu = await d.find('cm_customer', { where: { region: 'EU' } }); + expect(eu.map((r: any) => r.name).sort()).toEqual(['Borealis', 'Cyan']); + // array criterion: value -> ltv + const big = await d.find('cm_customer', { where: [['value', '>', 200]] }); + expect(big.map((r: any) => r.name).sort()).toEqual(['Aurora', 'Borealis']); + // mongo operator: value $gte + const gte = await d.find('cm_customer', { where: { value: { $gte: 312 } } }); + expect(gte.map((r: any) => r.name).sort()).toEqual(['Aurora', 'Borealis']); + } finally { await d.disconnect(); } + }); + + it('ORDER BY + date coercion work through the columnMap', async () => { + const d = ext(file); + await d.connect(); + try { + d.registerExternalObject({ name: 'cm_customer', external: { remoteName: 'legacy_cust', columnMap: COLUMN_MAP }, fields: LOCAL_FIELDS }); + // orderBy value (-> ltv) desc + const ordered = await d.find('cm_customer', { orderBy: [{ field: 'value', order: 'desc' }] }); + expect(ordered.map((r: any) => r.value)).toEqual([480, 312, 95]); + // date filter: signed_up (-> signup_dt) coercion keyed by local field + const onDate = await d.find('cm_customer', { where: { signed_up: '2026-02-15' } }); + expect(onDate.map((r: any) => r.name)).toEqual(['Borealis']); + } finally { await d.disconnect(); } + }); + + it('managed objects (no columnMap) are unaffected — own column names', async () => { + const d = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) as any; + d.name = 'default'; + await d.connect(); + try { + await d.initObjects([{ name: 'plain', fields: { title: { type: 'text' }, qty: { type: 'number' } } }]); + await d.create('plain', { id: 'p1', title: 'Widget', qty: 5 }); + const rows = await d.find('plain', { where: { title: 'Widget' }, orderBy: [{ field: 'qty', order: 'asc' }] }); + expect(rows).toHaveLength(1); + expect(rows[0].title).toBe('Widget'); + expect(rows[0].qty).toBe(5); + } finally { await d.disconnect(); } + }); +}); diff --git a/packages/plugins/driver-sql/src/sql-driver.ts b/packages/plugins/driver-sql/src/sql-driver.ts index 04cdc20828..70d328f930 100644 --- a/packages/plugins/driver-sql/src/sql-driver.ts +++ b/packages/plugins/driver-sql/src/sql-driver.ts @@ -207,6 +207,10 @@ export class SqlDriver implements IDataDriver { protected physicalTableByObject: Record = {}; protected physicalSchemaByObject: Record = {}; protected objectByPhysicalTable: Record = {}; + /** External columnMap (ADR-0015): logical field -> physical remote column (for WHERE/ORDER BY/writes). */ + protected fieldColumnByObject: Record> = {}; + /** External columnMap inverse: physical remote column -> logical field (for read output remap). */ + protected columnFieldByObject: Record> = {}; protected tablesWithTimestamps: Set = new Set(); /** * Autonumber field configs per table, captured during initObjects. @@ -453,7 +457,7 @@ export class SqlDriver implements IDataDriver { if (query.orderBy && Array.isArray(query.orderBy)) { for (const item of query.orderBy) { if (item.field) { - b.orderBy(this.mapSortField(item.field), item.order || 'asc'); + b.orderBy(this.remoteColumn(object, item.field, this.mapSortField(item.field)), item.order || 'asc'); } } } @@ -567,7 +571,7 @@ export class SqlDriver implements IDataDriver { await this.fillAutoNumberFields(object, toInsert, options); const builder = this.getBuilder(object, options); - const formatted = this.formatInput(object, toInsert); + const formatted = this.applyWriteColumnMap(object, this.formatInput(object, toInsert)); const result = await builder.insert(formatted).returning('*'); return this.formatOutput(object, result[0]); @@ -881,7 +885,7 @@ export class SqlDriver implements IDataDriver { this.auditMissingTenant(object, 'update', options); const builder = this.getBuilder(object, options).where('id', id); this.applyTenantScope(builder, object, options); - const formatted = this.formatInput(object, data); + const formatted = this.applyWriteColumnMap(object, this.formatInput(object, data)); if (this.tablesWithTimestamps.has(object)) { if (this.isSqlite) { @@ -914,7 +918,7 @@ export class SqlDriver implements IDataDriver { this.injectTenantOnInsert(object, toUpsert, options); await this.fillAutoNumberFields(object, toUpsert, options); - const formatted = this.formatInput(object, toUpsert); + const formatted = this.applyWriteColumnMap(object, this.formatInput(object, toUpsert)); const mergeKeys = conflictKeys && conflictKeys.length > 0 ? conflictKeys : ['id']; const builder = this.getBuilder(object, options); @@ -1283,7 +1287,7 @@ export class SqlDriver implements IDataDriver { name: string; fields?: Record; tenancy?: any; - external?: { remoteName?: string; remoteSchema?: string }; + external?: { remoteName?: string; remoteSchema?: string; columnMap?: Record }; }): void { const key = schema.name; const remoteName = schema.external?.remoteName || schema.name; @@ -1300,6 +1304,23 @@ export class SqlDriver implements IDataDriver { } } + // External columnMap (ADR-0015) is declared as { remoteColumn -> localField }. + // Keep it for read-output remap, and invert to { localField -> remoteColumn } + // for WHERE/ORDER BY/write translation. Absent => managed-identical behavior. + const columnMap = schema.external?.columnMap; + if (columnMap && typeof columnMap === 'object' && Object.keys(columnMap).length > 0) { + const fieldToCol: Record = {}; + const colToField: Record = {}; + for (const [remoteCol, localField] of Object.entries(columnMap)) { + if (typeof localField === 'string' && localField) { + fieldToCol[localField] = remoteCol; + colToField[remoteCol] = localField; + } + } + this.fieldColumnByObject[key] = fieldToCol; + this.columnFieldByObject[key] = colToField; + } + const jsonCols: string[] = []; const booleanCols: string[] = []; const numericCols: string[] = []; @@ -1944,7 +1965,7 @@ export class SqlDriver implements IDataDriver { for (const [key, value] of Object.entries(filters)) { if (['limit', 'offset', 'fields', 'orderBy'].includes(key)) continue; - builder.where(key, this.coerceFilterValue(table, key, value) as any); + builder.where(this.remoteColumn(table, key, key), this.coerceFilterValue(table, key, value) as any); } return; } @@ -1965,8 +1986,9 @@ export class SqlDriver implements IDataDriver { const isCriterion = typeof fieldRaw === 'string' && typeof op === 'string'; if (isCriterion) { - const field = this.mapSortField(fieldRaw); - const coerced = this.coerceFilterValue(table, field, value); + const localField = this.mapSortField(fieldRaw); + const field = this.remoteColumn(table, fieldRaw, localField); + const coerced = this.coerceFilterValue(table, localField, value); const apply = (b: any) => { const method = nextJoin === 'or' ? 'orWhere' : 'where'; const methodIn = nextJoin === 'or' ? 'orWhereIn' : 'whereIn'; @@ -2044,10 +2066,11 @@ export class SqlDriver implements IDataDriver { } }); } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) { - const field = this.mapSortField(key); + const localField = this.mapSortField(key); + const field = this.remoteColumn(table, key, localField); for (const [op, opValue] of Object.entries(value as Record)) { const method = logicalOp === 'or' ? 'orWhere' : 'where'; - const coerced = this.coerceFilterValue(table, field, opValue); + const coerced = this.coerceFilterValue(table, localField, opValue); switch (op) { case '$eq': (builder as any)[method](field, coerced); @@ -2085,9 +2108,10 @@ export class SqlDriver implements IDataDriver { } } } else { - const field = this.mapSortField(key); + const localField = this.mapSortField(key); + const field = this.remoteColumn(table, key, localField); const method = logicalOp === 'or' ? 'orWhere' : 'where'; - (builder as any)[method](field, this.coerceFilterValue(table, field, value) as any); + (builder as any)[method](field, this.coerceFilterValue(table, localField, value) as any); } } } @@ -2100,6 +2124,30 @@ export class SqlDriver implements IDataDriver { return field; } + /** + * Physical column for a logical field on an external object that declares an + * `external.columnMap` (ADR-0015). Returns `fallback` (the caller's existing + * per-site resolution) when the object has no columnMap, so managed objects + * and external objects without a columnMap are byte-for-byte unchanged. + */ + protected remoteColumn(object: string | null | undefined, field: string, fallback: string): string { + const m = object ? this.fieldColumnByObject[object] : undefined; + return (m && m[field]) || fallback; + } + + /** + * Remap a write payload's logical field keys to physical remote columns for an + * external object with a columnMap. No-op otherwise. Applied AFTER formatInput + * (whose value coercion is keyed by logical field name). + */ + protected applyWriteColumnMap(object: string, data: any): any { + const m = this.fieldColumnByObject[object]; + if (!m || !data || typeof data !== 'object') return data; + const out: any = {}; + for (const [k, v] of Object.entries(data)) out[m[k] ?? k] = v; + return out; + } + protected mapAggregateFunc(func: string): string { switch (func) { case 'count': @@ -2412,6 +2460,21 @@ export class SqlDriver implements IDataDriver { protected formatOutput(object: string, data: any): any { if (!data) return data; + // External columnMap (ADR-0015): rename physical remote-column keys to local + // field names BEFORE coercion (which is keyed by local field). No-op for + // managed objects and external objects without a columnMap. + const colToField = this.columnFieldByObject[object]; + if (colToField && typeof data === 'object') { + for (const [remoteCol, localField] of Object.entries(colToField)) { + if (remoteCol !== localField && Object.prototype.hasOwnProperty.call(data, remoteCol)) { + // Explicit columnMap wins: the remote column is the source of truth for + // this local field, even if a same-named native column also exists. + data[localField] = data[remoteCol]; + delete data[remoteCol]; + } + } + } + if (this.isSqlite) { const jsonFields = this.jsonFields[object]; if (jsonFields && jsonFields.length > 0) {