Skip to content

Commit bea8835

Browse files
committed
Add a stamped boot-time data-migration ledger for the libSQL apps
Before this, every boot-time data fix (auth-config placements, openapi envelope unwrap) re-scanned its tables on every startup to decide whether it had already run, inferring state from data shape. That accumulates — N migrations means N full-table scans per boot forever — and makes idempotence a per-migration proof obligation. Now the sdk owns a data_migration ledger (name -> completion time) and an Effect-native runner: apps compose an ordered registry of named migrations; pending ones run once and are stamped, stamped ones are skipped without touching data. Failures are typed (DataMigrationError / DuplicateDataMigrationError) and fail the boot without stamping. apps/local and apps/host-selfhost convert their two existing boot scans into the first registry entries. The openapi unwrap migration is rewritten Effect-native with the same envelope detection. Registry names are append-only and never renamed. Verified end-to-end against a real libSQL database: first boot applies both migrations and unwraps a seeded envelope row (payload rows untouched), second boot applies nothing, ledger holds both stamps. Also fixes a cloud test still asserting the retired envelope shape (missed in the payload-first change; the turbo test pipeline did not propagate that failure).
1 parent fed3151 commit bea8835

10 files changed

Lines changed: 476 additions & 130 deletions

File tree

apps/host-selfhost/src/app.ts

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
import { HttpApiSwagger } from "effect/unstable/httpapi";
22
import { HttpEffect, HttpRouter } from "effect/unstable/http";
3-
import { Layer } from "effect";
3+
import { Effect, Layer } from "effect";
44

55
import { composePluginApi, ExecutorApp, textFailureStrategy } from "@executor-js/api/server";
66

7-
import { runSqliteAuthConfigMigration } from "@executor-js/sdk/http-auth";
8-
import { runSqliteOpenApiOutputSchemaMigration } from "@executor-js/plugin-openapi";
7+
import { runSqliteDataMigrations } from "@executor-js/sdk";
98

109
import { resolveAuthProviders } from "./auth";
11-
import { authConfigTransforms } from "./db/auth-config-migration";
10+
import { selfHostDataMigrations } from "./db/data-migrations";
1211
import { makeSelfHostAdminApiLayer } from "./admin/handlers";
1312
import { makeSelfHostSystemApiLayer } from "./system/handlers";
1413
import { selfHostAccountMiddleware } from "./account";
@@ -58,15 +57,10 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => {
5857
version: SELF_HOST_SCHEMA_VERSION,
5958
});
6059

61-
// One-off data migration: rewrite pre-canonical integration auth configs
62-
// into the shared placements model. Idempotent — a no-op once every row is
63-
// canonical (same defensive-at-boot pattern as the column adds above).
64-
await runSqliteAuthConfigMigration(dbHandle.client, authConfigTransforms);
65-
66-
// One-off data migration: unwrap the retired {status, headers, data}
67-
// transport envelope from persisted openapi tool output schemas (mirrors
68-
// cloud's drizzle 0002). Idempotent — payload-shaped rows don't match.
69-
await runSqliteOpenApiOutputSchemaMigration(dbHandle.client);
60+
// Boot-time data migrations: each registry entry runs once and is stamped
61+
// in the `data_migration` ledger; stamped entries are skipped without
62+
// touching the data.
63+
await Effect.runPromise(runSqliteDataMigrations(dbHandle.client, selfHostDataMigrations));
7064

7165
// ---- auth providers ---------------------------------------------------
7266
// Better Auth: cookie/bearer/api-key identity + /api/auth handler + account
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// ---------------------------------------------------------------------------
2+
// The ordered boot-time data-migration registry for the selfhost app.
3+
// Entries run once and are stamped in the `data_migration` ledger (see
4+
// @executor-js/sdk sqlite-data-migrations). Names are append-only and never
5+
// renamed.
6+
// ---------------------------------------------------------------------------
7+
8+
import { sqliteDataMigration, type SqliteDataMigration } from "@executor-js/sdk";
9+
import { runSqliteAuthConfigMigration } from "@executor-js/sdk/http-auth";
10+
import { openApiOutputSchemaDataMigration } from "@executor-js/plugin-openapi";
11+
12+
import { authConfigTransforms } from "./auth-config-migration";
13+
14+
export const selfHostDataMigrations: readonly SqliteDataMigration[] = [
15+
// Rewrite pre-canonical integration auth configs into the shared
16+
// placements model.
17+
sqliteDataMigration("2026-06-05-auth-config-placements", (client) =>
18+
runSqliteAuthConfigMigration(client, authConfigTransforms),
19+
),
20+
// Unwrap the retired {status, headers, data} transport envelope from
21+
// persisted openapi tool output schemas (mirrors cloud's drizzle 0002).
22+
openApiOutputSchemaDataMigration,
23+
];
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// ---------------------------------------------------------------------------
2+
// The ordered boot-time data-migration registry for the local app. Entries
3+
// run once and are stamped in the `data_migration` ledger (see
4+
// @executor-js/sdk sqlite-data-migrations). Names are append-only and never
5+
// renamed.
6+
// ---------------------------------------------------------------------------
7+
8+
import { sqliteDataMigration, type SqliteDataMigration } from "@executor-js/sdk";
9+
import { runSqliteAuthConfigMigration } from "@executor-js/sdk/http-auth";
10+
import { openApiOutputSchemaDataMigration } from "@executor-js/plugin-openapi";
11+
12+
import { authConfigTransforms } from "./auth-config-migration";
13+
14+
export const localDataMigrations: readonly SqliteDataMigration[] = [
15+
// Rewrite pre-canonical integration auth configs (incl. v1→v2 outputs)
16+
// into the shared placements model.
17+
sqliteDataMigration("2026-06-05-auth-config-placements", (client) =>
18+
runSqliteAuthConfigMigration(client, authConfigTransforms),
19+
),
20+
// Unwrap the retired {status, headers, data} transport envelope from
21+
// persisted openapi tool output schemas (mirrors cloud's drizzle 0002).
22+
openApiOutputSchemaDataMigration,
23+
];

apps/local/src/executor.ts

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,19 @@ import { homedir } from "node:os";
44
import { basename, join } from "node:path";
55
import { createHash } from "node:crypto";
66

7-
import { Subject, Tenant, createExecutor, type AnyPlugin, type Executor } from "@executor-js/sdk";
8-
import { runSqliteAuthConfigMigration } from "@executor-js/sdk/http-auth";
9-
import { runSqliteOpenApiOutputSchemaMigration } from "@executor-js/plugin-openapi";
7+
import {
8+
Subject,
9+
Tenant,
10+
createExecutor,
11+
runSqliteDataMigrations,
12+
type AnyPlugin,
13+
type Executor,
14+
} from "@executor-js/sdk";
1015
import { collectTables } from "@executor-js/api/server";
1116
import { loadPluginsFromJsonc } from "@executor-js/config";
1217

1318
import executorConfig from "../executor.config";
14-
import { authConfigTransforms } from "./db/auth-config-migration";
19+
import { localDataMigrations } from "./db/data-migrations";
1520
import { createSqliteFumaDb } from "./db/sqlite-fumadb";
1621
import { migrateLocalV1ToV2IfNeeded } from "./db/v1-v2-migration";
1722

@@ -170,24 +175,14 @@ const createLocalExecutorLayer = () => {
170175
(db) => Effect.promise(() => db.close()).pipe(Effect.ignore),
171176
);
172177

173-
// One-off data migration: rewrite pre-canonical integration auth
174-
// configs (incl. v1→v2 outputs) into the shared placements model.
175-
// Idempotent — a no-op once every row is canonical.
176-
yield* Effect.tryPromise({
177-
try: () => runSqliteAuthConfigMigration(sqlite.client, authConfigTransforms),
178-
catch: (cause) =>
179-
new LocalExecutorCreateError({ message: CREATE_SQLITE_ERROR_MESSAGE, cause }),
180-
});
181-
182-
// One-off data migration: unwrap the retired {status, headers, data}
183-
// transport envelope from persisted openapi tool output schemas
184-
// (mirrors cloud's drizzle 0002). Idempotent — payload-shaped rows
185-
// don't match the envelope signature.
186-
yield* Effect.tryPromise({
187-
try: () => runSqliteOpenApiOutputSchemaMigration(sqlite.client),
188-
catch: (cause) =>
189-
new LocalExecutorCreateError({ message: CREATE_SQLITE_ERROR_MESSAGE, cause }),
190-
});
178+
// Boot-time data migrations: each registry entry runs once and is
179+
// stamped in the `data_migration` ledger; stamped entries are skipped
180+
// without touching the data.
181+
yield* runSqliteDataMigrations(sqlite.client, localDataMigrations).pipe(
182+
Effect.mapError(
183+
(cause) => new LocalExecutorCreateError({ message: CREATE_SQLITE_ERROR_MESSAGE, cause }),
184+
),
185+
);
191186

192187
// webBaseUrl is where the executor's web UI listens — same port as the
193188
// daemon API since the daemon serves both. Mirrors serve.ts's port

packages/core/sdk/src/index.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,16 @@ export { InternalError } from "./api-errors";
324324

325325
// ToolResult — typed value-based discriminated union for tool outcomes.
326326
export { ToolResult, isToolResult, type ToolError, type ToolHttpMeta } from "./tool-result";
327+
328+
// Stamped boot-time data-migration ledger for the libSQL-backed apps.
329+
export {
330+
DataMigrationError,
331+
DuplicateDataMigrationError,
332+
runSqliteDataMigrations,
333+
sqliteDataMigration,
334+
type SqliteDataMigration,
335+
type SqliteDataMigrationClient,
336+
} from "./sqlite-data-migrations";
327337
export {
328338
authToolFailure,
329339
type AuthToolFailureCode,
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
import { Effect, Predicate } from "effect";
3+
4+
import {
5+
DataMigrationError,
6+
runSqliteDataMigrations,
7+
sqliteDataMigration,
8+
type SqliteDataMigration,
9+
type SqliteDataMigrationClient,
10+
} from "./sqlite-data-migrations";
11+
12+
// A tiny scripted fake standing in for a libSQL client: tracks statements
13+
// and simulates the ledger table.
14+
const makeFakeClient = (stampedNames: string[]) => {
15+
const log: unknown[] = [];
16+
const stamps = [...stampedNames];
17+
const client: SqliteDataMigrationClient = {
18+
execute: (stmt) => {
19+
log.push(stmt);
20+
if (typeof stmt === "string" && stmt.startsWith("SELECT name FROM data_migration")) {
21+
return Promise.resolve({ rows: stamps.map((name) => ({ name })) });
22+
}
23+
if (typeof stmt === "object" && stmt.sql.startsWith("INSERT INTO data_migration")) {
24+
stamps.push(String(stmt.args[0]));
25+
}
26+
return Promise.resolve({ rows: [] });
27+
},
28+
};
29+
return { client, log, stamps };
30+
};
31+
32+
const migrationSpy = (name: string) => {
33+
const calls: number[] = [];
34+
const migration: SqliteDataMigration = {
35+
name,
36+
run: () => Effect.sync(() => void calls.push(calls.length)),
37+
};
38+
return { migration, calls };
39+
};
40+
41+
describe("runSqliteDataMigrations", () => {
42+
it.effect("runs pending migrations in order and stamps them", () =>
43+
Effect.gen(function* () {
44+
const { client, log, stamps } = makeFakeClient([]);
45+
const a = migrationSpy("2026-06-05-a");
46+
const b = migrationSpy("2026-06-11-b");
47+
const applied = yield* runSqliteDataMigrations(client, [a.migration, b.migration]);
48+
expect(applied).toEqual(["2026-06-05-a", "2026-06-11-b"]);
49+
expect(a.calls.length).toBe(1);
50+
expect(b.calls.length).toBe(1);
51+
expect(stamps).toEqual(["2026-06-05-a", "2026-06-11-b"]);
52+
expect(String(log[0])).toContain("CREATE TABLE IF NOT EXISTS data_migration");
53+
}),
54+
);
55+
56+
it.effect("skips stamped migrations without running them", () =>
57+
Effect.gen(function* () {
58+
const { client } = makeFakeClient(["2026-06-05-a"]);
59+
const a = migrationSpy("2026-06-05-a");
60+
const b = migrationSpy("2026-06-11-b");
61+
const applied = yield* runSqliteDataMigrations(client, [a.migration, b.migration]);
62+
expect(applied).toEqual(["2026-06-11-b"]);
63+
expect(a.calls.length).toBe(0);
64+
expect(b.calls.length).toBe(1);
65+
}),
66+
);
67+
68+
it.effect("is a no-op once everything is stamped", () =>
69+
Effect.gen(function* () {
70+
const { client } = makeFakeClient([]);
71+
const a = migrationSpy("2026-06-05-a");
72+
yield* runSqliteDataMigrations(client, [a.migration]);
73+
const applied = yield* runSqliteDataMigrations(client, [a.migration]);
74+
expect(applied).toEqual([]);
75+
expect(a.calls.length).toBe(1);
76+
}),
77+
);
78+
79+
it.effect("a failing migration leaves no stamp and surfaces the failure", () =>
80+
Effect.gen(function* () {
81+
const { client, stamps } = makeFakeClient([]);
82+
const boom: SqliteDataMigration = {
83+
name: "2026-06-11-boom",
84+
run: () =>
85+
Effect.fail(new DataMigrationError({ migration: "2026-06-11-boom", cause: "nope" })),
86+
};
87+
const after = migrationSpy("2026-06-12-after");
88+
const failure = yield* runSqliteDataMigrations(client, [boom, after.migration]).pipe(
89+
Effect.flip,
90+
);
91+
expect(Predicate.isTagged(failure, "DataMigrationError")).toBe(true);
92+
expect(stamps).toEqual([]);
93+
expect(after.calls.length).toBe(0);
94+
}),
95+
);
96+
97+
it.effect("rejects duplicate names before touching the database", () =>
98+
Effect.gen(function* () {
99+
const { client, log } = makeFakeClient([]);
100+
const a1 = migrationSpy("2026-06-05-a");
101+
const a2 = migrationSpy("2026-06-05-a");
102+
const failure = yield* runSqliteDataMigrations(client, [a1.migration, a2.migration]).pipe(
103+
Effect.flip,
104+
);
105+
expect(Predicate.isTagged(failure, "DuplicateDataMigrationError")).toBe(true);
106+
expect(log).toEqual([]);
107+
}),
108+
);
109+
110+
it.effect("sqliteDataMigration adapts promise-shaped bodies with typed failures", () =>
111+
Effect.gen(function* () {
112+
const { client, stamps } = makeFakeClient([]);
113+
let ran = 0;
114+
const ok = sqliteDataMigration("2026-06-05-promise", () => {
115+
ran++;
116+
return Promise.resolve(42);
117+
});
118+
yield* runSqliteDataMigrations(client, [ok]);
119+
expect(ran).toBe(1);
120+
expect(stamps).toEqual(["2026-06-05-promise"]);
121+
122+
// oxlint-disable-next-line executor/no-promise-reject -- simulates a raw driver rejection at the adapter boundary under test
123+
const bad = sqliteDataMigration("2026-06-12-bad", () => Promise.reject("disk full"));
124+
const failure = yield* runSqliteDataMigrations(client, [ok, bad]).pipe(Effect.flip);
125+
expect(Predicate.isTagged(failure, "DataMigrationError")).toBe(true);
126+
expect((failure as DataMigrationError).migration).toBe("2026-06-12-bad");
127+
expect(ran).toBe(1);
128+
}),
129+
);
130+
});

0 commit comments

Comments
 (0)