diff --git a/.changeset/adr-0104-advertise-open-gates.md b/.changeset/adr-0104-advertise-open-gates.md new file mode 100644 index 0000000000..a418fb1372 --- /dev/null +++ b/.changeset/adr-0104-advertise-open-gates.md @@ -0,0 +1,34 @@ +--- +"@objectstack/objectql": patch +"@objectstack/cli": patch +--- + +feat(migrate,objectql): the upgrade path names the data migrations that are still open here (#3438, ADR-0104 2026-07-30) + +Both value-shape gates fail toward leniency: a deployment that never runs its +migration keeps warning instead of rejecting, and keeps every released file +forever. That default is right — and completely silent, so the gate could sit +open for the life of a deployment without anyone learning that one command ends +it. A gate nobody is told about is served by nobody. + +Two announcements, each where an upgrade actually looks: + +- **`os migrate meta --from 16`** now ends by naming the data migrations a + chain crossing into 17 leaves behind — `files-to-references`, `value-shapes` + — with what each unlocks, scoped to the field classes the author's own + metadata declares (an app with no media field is never told about the file + migration). `--json` carries the same list as `dataMigrations`. The command + reads no database, so it reports what remains *to do*, never what a given + deployment has *done*. +- **The server logs one line per open gate at boot**, naming the command that + closes it. Only the lax posture announces itself — a verified gate already + logs that it is enforcing, and an app declaring neither class of field costs + nothing and says nothing. This is the half that can speak to a deployment's + actual data, because it is the half with the database. + +Nothing about enforcement changes: same gates, same flags, same fail-toward- +leniency default. The advisory runs on `kernel:bootstrapped` rather than +`kernel:ready`, deliberately — the answer depends on the storage service's own +ready handler, which registers `sys_migration` and may attest a store it just +created, and racing it would tell a brand-new deployment its gates are open +moments after they closed. diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index e9780c8037..9f9be95817 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -745,6 +745,22 @@ them temporarily, `OS_ALLOW_LAX_MEDIA_VALUES=1` (media) and per class, and re-running the corresponding migration re-establishes its flag from the data itself. +#### How you find out a gate is open + +You are told, in the two places an upgrade actually looks. + +**`os migrate meta --from 16`** — the metadata half of the upgrade — ends by +naming the data migrations that remain, scoped to the field classes *your* +metadata declares. It reads no database, so it reports what is left to do, +never what this deployment has already done. The machine-readable output +carries the same list under `dataMigrations`. + +**The server, once per boot**, logs one line per gate that is still open here +and the command that closes it. Only the lax posture announces itself: a +closed gate logs that it is enforcing, and an app that declares neither class +of field says nothing at all. So a running deployment always tells you the +state of its own data — which is the question `os migrate meta` cannot answer. + ### Scaffolding | Command | Alias | Description | diff --git a/packages/cli/src/commands/migrate/meta.ts b/packages/cli/src/commands/migrate/meta.ts index 7d74570e73..f9c0f81a3b 100644 --- a/packages/cli/src/commands/migrate/meta.ts +++ b/packages/cli/src/commands/migrate/meta.ts @@ -12,6 +12,8 @@ import { MigrationFloorError, } from '@objectstack/spec'; import { PROTOCOL_MAJOR, PROTOCOL_VERSION } from '@objectstack/spec/kernel'; +import { FILE_REFERENCE_TYPES, REFERENCE_VALUE_TYPES, STRUCTURED_JSON_TYPES } from '@objectstack/spec/data'; +import { FILE_REFERENCES_MIGRATION_ID, VALUE_SHAPES_MIGRATION_ID } from '@objectstack/spec/system'; import { loadConfig } from '../../utils/config.js'; import { printHeader, @@ -24,6 +26,86 @@ import { emitJson, } from '../../utils/format.js'; +/** The protocol major that introduced the per-deployment value-shape gates. */ +const VALUE_SHAPE_GATE_MAJOR = 17; + +interface PendingDataMigration { + /** `sys_migration` row id the run records. */ + id: string; + command: string; + /** What staying un-run costs, in this deployment's terms. */ + unlocks: string; +} + +/** + * The DATA migrations a metadata chain crossing into {@link VALUE_SHAPE_GATE_MAJOR} + * leaves for the operator (ADR-0104's 2026-07-30 addendum, #3438). + * + * Metadata migration and data migration are different jobs with different + * subjects: this command rewrites an author's source, while these two rewrite + * (or vouch for) a deployment's rows, one deployment at a time. Nothing here + * can run them, and — with no database in reach — nothing here can say whether + * they have run; the booting server reports that. What this can do is make + * sure the upgrade never *ends* without naming them, because a gate nobody is + * told about is served by nobody. + * + * Listed only when the author's own metadata declares the field classes each + * gate is about, so the advice is never noise. + */ +function pendingDataMigrations(stack: any, fromMajor: number, toMajor: number): PendingDataMigration[] { + if (!(fromMajor < VALUE_SHAPE_GATE_MAJOR && toMajor >= VALUE_SHAPE_GATE_MAJOR)) return []; + + let media = false; + let covered = false; + for (const obj of (Array.isArray(stack?.objects) ? stack.objects : []) as any[]) { + for (const def of Object.values(obj?.fields ?? {}) as any[]) { + if (!def?.type) continue; + if (FILE_REFERENCE_TYPES.has(def.type)) media = true; + else if (REFERENCE_VALUE_TYPES.has(def.type) || STRUCTURED_JSON_TYPES.has(def.type)) covered = true; + } + if (media && covered) break; + } + + const pending: PendingDataMigration[] = []; + if (media) { + pending.push({ + id: FILE_REFERENCES_MIGRATION_ID, + command: 'os migrate files-to-references', + unlocks: + 'converts legacy file values to sys_file references, then enforces media value shapes ' + + 'and lets released files be collected. Until it passes here, media values only warn and ' + + 'released files are kept forever.', + }); + } + if (covered) { + pending.push({ + id: VALUE_SHAPES_MIGRATION_ID, + command: 'os migrate value-shapes', + unlocks: + 'scans stored reference and structured-JSON values against their field contracts. ' + + 'Until it passes here, a malformed value only warns.', + }); + } + return pending; +} + +/** Print the data-migration advice — the last thing a crossing upgrade sees. */ +function printPendingDataMigrations(pending: PendingDataMigration[]): void { + if (pending.length === 0) return; + console.log(chalk.bold(' Then, against each deployment\'s database:')); + for (const m of pending) { + console.log(` ${chalk.cyan('→')} ${chalk.white(m.command)}`); + console.log(chalk.dim(` ${m.unlocks}`)); + } + console.log( + chalk.dim( + ' Both are dry-run by default and report what they would do; `--apply` is the only ' + + 'writing mode. Not running them is safe — enforcement simply stays off here.', + ), + ); + console.log(''); +} + /** * `os migrate meta --from N` — replay the ADR-0087 D3 migration chain. * @@ -94,6 +176,7 @@ export default class MigrateMeta extends Command { // diff" the consumer agent reviews (ADR-0087 D3/D5). const parsed = ObjectStackDefinitionSchema.safeParse(result.stack); const specChanges = composeSpecChanges(flags.from, toMajor); + const dataMigrations = pendingDataMigrations(result.stack, result.fromMajor, result.toMajor); if (flags.json) { await emitJson({ @@ -112,6 +195,9 @@ export default class MigrateMeta extends Command { : undefined, specChanges, schemaValid: parsed.success, + // Per-deployment data migrations this chain leaves to the + // operator — the metadata is only half of a crossing upgrade. + dataMigrations, duration: timer.elapsed(), }); if (flags.out) writeFileSync(resolve(flags.out), JSON.stringify(result.stack, null, 2)); @@ -124,6 +210,10 @@ export default class MigrateMeta extends Command { if (result.applied.length === 0 && result.todos.length === 0) { printSuccess('Nothing to migrate — the metadata is already canonical for this range.'); + // Still advertise: metadata needing no rewrite says nothing about + // whether this deployment's DATA has been migrated. + console.log(''); + printPendingDataMigrations(dataMigrations); return; } @@ -162,6 +252,8 @@ export default class MigrateMeta extends Command { printInfo(`Wrote migrated stack snapshot → ${chalk.white(resolve(flags.out))}`); } + printPendingDataMigrations(dataMigrations); + if (parsed.success) { printSuccess(`Migrated stack is schema-valid ${chalk.dim(`(${timer.display()})`)}`); } else { diff --git a/packages/cli/test/migrate-meta.e2e.test.ts b/packages/cli/test/migrate-meta.e2e.test.ts index b32893d1c9..05a63e1816 100644 --- a/packages/cli/test/migrate-meta.e2e.test.ts +++ b/packages/cli/test/migrate-meta.e2e.test.ts @@ -230,6 +230,76 @@ describe('os migrate meta --from 16 (e2e over the real CLI)', () => { } }, 120_000); + /** + * ADR-0104 / #3438. The metadata half of a 16→17 upgrade is only half of it: + * two DATA migrations gate enforcement per deployment, and a gate nobody is + * told about is served by nobody. The advice is scoped to the field classes + * the author actually declares, so it is never noise. + */ + describe('per-deployment data migrations (#3438)', () => { + const withFields = (fields: string) => ` + export default { + name: 'dm_probe', label: 'DM Probe', + objects: [{ name: 'dm_thing', label: 'Thing', fields: { ${fields} } }], + }; + `; + + async function runJson(fields: string, args: string[] = ['--from', '16', '--json']) { + const d = mkdtempSync(join(tmpdir(), 'os-migrate-meta-dm-')); + try { + writeFileSync(join(d, 'objectstack.config.ts'), withFields(fields)); + return JSON.parse(await runMeta(args, d)); + } finally { + try { rmSync(d, { recursive: true, force: true }); } catch { /* ignore */ } + } + } + + it('names the file migration when the metadata declares a media field', async () => { + const res = await runJson(`cover: { type: 'image', label: 'Cover' }`); + expect(res.dataMigrations.map((m: any) => m.id)).toEqual(['adr-0104-file-references']); + }, 120_000); + + it('names the value-shape scan when the metadata declares a covered field', async () => { + const res = await runJson(`spot: { type: 'location', label: 'Spot' }`); + expect(res.dataMigrations.map((m: any) => m.id)).toEqual(['adr-0104-value-shapes']); + }, 120_000); + + it('names both when both classes are declared', async () => { + const res = await runJson( + `cover: { type: 'image', label: 'Cover' }, spot: { type: 'location', label: 'Spot' }`, + ); + expect(res.dataMigrations.map((m: any) => m.id).sort()).toEqual([ + 'adr-0104-file-references', + 'adr-0104-value-shapes', + ]); + }, 120_000); + + it('stays silent for metadata that declares neither class', async () => { + const res = await runJson(`title: { type: 'text', label: 'Title' }`); + expect(res.dataMigrations).toEqual([]); + }, 120_000); + + it('says nothing on a chain that does not cross into 17', async () => { + const res = await runJson(`cover: { type: 'image', label: 'Cover' }`, ['--from', '17', '--json']); + expect(res.dataMigrations).toEqual([]); + }, 120_000); + + it('ends the human-readable upgrade with the advice, not just the JSON', async () => { + // Where an operator actually reads it. The advice names the command and + // says the run is dry by default, so nothing here reads as "do this and + // something irreversible happens". + const d = mkdtempSync(join(tmpdir(), 'os-migrate-meta-dm-h-')); + try { + writeFileSync(join(d, 'objectstack.config.ts'), withFields(`cover: { type: 'image', label: 'Cover' }`)); + const stdout = await runMeta(['--from', '16'], d); + expect(stdout).toMatch(/os migrate files-to-references/); + expect(stdout).toMatch(/dry-run by default/); + } finally { + try { rmSync(d, { recursive: true, force: true }); } catch { /* ignore */ } + } + }, 120_000); + }); + it('refuses a --from below the support floor with the structured error', async () => { let failed = false; try { diff --git a/packages/objectql/src/engine.test.ts b/packages/objectql/src/engine.test.ts index 9f1f22e946..dfa0b799cb 100644 --- a/packages/objectql/src/engine.test.ts +++ b/packages/objectql/src/engine.test.ts @@ -18,6 +18,7 @@ vi.mock('./registry', () => { const instance: any = { getObject: vi.fn((name) => mockObjects.get(name)), resolveObject: vi.fn((name) => mockObjects.get(name)), + getAllObjects: vi.fn(() => [...mockObjects.values()]), registerObject: vi.fn((obj, packageId, namespace, ownership, priority) => { const fqn = namespace ? `${namespace}__${obj.name}` : obj.name; mockObjects.set(fqn, { ...obj, name: fqn }); @@ -2318,4 +2319,96 @@ describe('ObjectQL — file-as-reference migration flag (#3617)', () => { expect(engine.wasDatastoreCreatedFromEmpty()).toBe(false); }); }); + + /** + * ADR-0104 / #3438. Both gates fail toward leniency, which is right and + * silent — so the LAX posture is the one that has to announce itself, or the + * deployment never learns a command would end it. Scoped to what this + * deployment's own metadata declares, so it is never noise. + */ + describe('announceOpenMigrationGates', () => { + const registerObjects = (objects: Record) => { + vi.mocked(SchemaRegistry.getObject).mockImplementation((name: string) => objects[name]); + vi.mocked((SchemaRegistry as any).getAllObjects).mockImplementation(() => Object.values(objects)); + }; + const SYS_MIGRATION = { name: 'sys_migration', fields: { id: { type: 'text' } } }; + const lines = (info: any) => info.mock.calls.map((c: any[]) => String(c[0])).join('\n'); + + it('names the command that closes an open value-shape gate', async () => { + registerObjects({ + place: { name: 'place', fields: { spot: { type: 'location' } } }, + sys_migration: SYS_MIGRATION, + }); + vi.mocked(driver.find).mockResolvedValue([]); // no flag row → gate open + const info = vi.spyOn((engine as any).logger, 'info'); + + await engine.announceOpenMigrationGates(); + + expect(lines(info)).toMatch(/os migrate value-shapes/); + }); + + it('names the file migration for an app that stores media', async () => { + registerObjects({ + note: { name: 'note', fields: { doc: { type: 'file' } } }, + sys_migration: SYS_MIGRATION, + }); + vi.mocked(driver.find).mockResolvedValue([]); + const info = vi.spyOn((engine as any).logger, 'info'); + + await engine.announceOpenMigrationGates(); + + const said = lines(info); + expect(said).toMatch(/os migrate files-to-references/); + expect(said).not.toMatch(/os migrate value-shapes/); + }); + + it('says nothing about a gate this deployment has closed', async () => { + registerObjects({ + note: { name: 'note', fields: { doc: { type: 'file' } } }, + sys_migration: SYS_MIGRATION, + }); + vi.mocked(driver.find).mockResolvedValue([verifiedRow] as any); + const info = vi.spyOn((engine as any).logger, 'info'); + + await engine.announceOpenMigrationGates(); + + expect(lines(info)).not.toMatch(/os migrate files-to-references/); + }); + + it('stays silent — and costs no query — for an app declaring neither class', async () => { + registerObjects({ + tag: { name: 'tag', fields: { label: { type: 'text' } } }, + sys_migration: SYS_MIGRATION, + }); + const info = vi.spyOn((engine as any).logger, 'info'); + + await engine.announceOpenMigrationGates(); + + expect(lines(info)).not.toMatch(/os migrate/); + expect(driver.find).not.toHaveBeenCalled(); + }); + + it('announces once per process, not once per caller', async () => { + registerObjects({ + place: { name: 'place', fields: { spot: { type: 'location' } } }, + sys_migration: SYS_MIGRATION, + }); + vi.mocked(driver.find).mockResolvedValue([]); + const info = vi.spyOn((engine as any).logger, 'info'); + + await engine.announceOpenMigrationGates(); + const first = info.mock.calls.length; + await engine.announceOpenMigrationGates(); + + expect(info.mock.calls.length).toBe(first); + }); + + it('never throws into a boot when the registry cannot be read', async () => { + vi.mocked((SchemaRegistry as any).getAllObjects).mockImplementation(() => { + throw new Error('registry exploded'); + }); + + await expect(engine.announceOpenMigrationGates()).resolves.toBeUndefined(); + }); + }); }); diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 5a2f33bab2..51c525ea64 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -2267,8 +2267,59 @@ export class ObjectQL implements IDataEngine { invalidateDataMigrationFlags(): void { this.fileReferencesMigrationVerified = null; this.valueShapesMigrationVerified = null; + this.migrationGatesAnnounced = false; } + /** + * Say, once at boot, which value-shape gates are still open here and what + * closes them (ADR-0104's 2026-07-30 addendum, #3438). + * + * Both gates fail toward leniency: a deployment that never runs its + * migration keeps warning instead of rejecting, and keeps every released + * file forever. That default is deliberate — but silent, and a gate nobody + * is told about is served by nobody. The LAX posture is the one worth + * announcing, so this logs only when a gate is open *and* this deployment's + * own metadata says the gate is about something it stores; the verified case + * already logs from the flag read. + * + * Costs one flag read per applicable gate (both memoized, and both already + * paid by the first write to such an object), and nothing at all for an app + * that declares neither class of field. + */ + async announceOpenMigrationGates(): Promise { + if (this.migrationGatesAnnounced) return; + this.migrationGatesAnnounced = true; + try { + let media = false; + let covered = false; + for (const obj of this._registry.getAllObjects() as any[]) { + if (!media && this.objectHasMediaField(obj)) media = true; + if (!covered && this.objectHasCoveredValueField(obj)) covered = true; + if (media && covered) break; + } + + if (media && !(await this.isFileReferencesMigrationVerified())) { + this.logger.info( + '[value-shape] media values are checked but NOT enforced here, and released files are ' + + 'never collected — this deployment has not verified its file migration. Run ' + + '`os migrate files-to-references` (dry run) to see what it would do, then `--apply` ' + + 'to close the gate (ADR-0104 / #3617).', + ); + } + if (covered && !(await this.isValueShapesMigrationVerified())) { + this.logger.info( + '[value-shape] reference and structured-JSON values are checked but NOT enforced here — ' + + 'this deployment has not verified its value-shape scan. Run `os migrate value-shapes` ' + + '(dry run) to see what it would report, then `--apply` to close the gate ' + + '(ADR-0104 / #3438).', + ); + } + } catch { + // An advisory must never be the reason a boot fails. + } + } + private migrationGatesAnnounced = false; + /** * Did this process CREATE the datastore it is talking to, from empty? * diff --git a/packages/objectql/src/plugin.ts b/packages/objectql/src/plugin.ts index 6ad295871d..7c69816b4c 100644 --- a/packages/objectql/src/plugin.ts +++ b/packages/objectql/src/plugin.ts @@ -359,6 +359,16 @@ export class ObjectQLPlugin implements Plugin { // No settings service — governance stays at declared defaults. } }); + // ADR-0104 / #3438: name the value-shape gates still open on THIS + // deployment, and the command that closes each. Deliberately on + // `kernel:bootstrapped` rather than `kernel:ready`: the answer depends on + // the storage service's OWN ready handler, which registers `sys_migration` + // and may attest a store it just created. Racing it inside `kernel:ready` + // would tell a brand-new deployment its gates are open moments after they + // were closed. + ctx.hook('kernel:bootstrapped', async () => { + await this.ql?.announceOpenMigrationGates(); + }); ctx.hook('metadata:reloaded', async (payload?: unknown) => { await this.resyncAuthoredHooks(ctx); await this.resyncAuthoredActions(ctx);