Skip to content

Commit 9344df2

Browse files
hotlongCopilot
andcommitted
feat(cloud): out-of-band schema migration to fit CF Workers cold-start budget
Cloudflare Containers gives an inbound Worker request ~30s of wallclock before the platform tears the DO invocation down. A fresh control-plane boot against a cold Neon DB needs to CREATE TABLE for every sys_* object (driver-sql does not yet implement batchSchemaSync), which routinely takes 30-60s — the container is killed mid-DDL on every request and never reaches listen(4000). Move DDL out-of-band: * ObjectQLPlugin gains skipSchemaSync option (falls back to OS_SKIP_SCHEMA_SYNC=1) — gates both syncRegisteredSchemas() calls. Hydration of sys_metadata still runs so custom user objects come up. * objectstack serve honors OS_MIGRATE_AND_EXIT=1 — after runtime.start() resolves it shuts the kernel down and exits cleanly. * New apps/cloud/scripts/migrate.ts loads .env.cloudflare.secrets, forces OS_SKIP_SCHEMA_SYNC=0 + OS_MIGRATE_AND_EXIT=1, and spawns objectstack serve --prebuilt --no-ui --no-server. Schema sync runs from the deploy machine which has no 30s budget. * CloudContainer ships with OS_SKIP_SCHEMA_SYNC=1 baked in (overridable via wrangler secret put). * deploy-cloudflare.sh inserts the migrate step between build and push, with --skip-migrate to opt out. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent e1cb036 commit 9344df2

7 files changed

Lines changed: 293 additions & 17 deletions

File tree

apps/cloud/cloudflare/worker.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ export interface Env {
6262
OS_DATA_DIR?: string;
6363
OS_PROVISION_SYNC?: string;
6464
OS_EAGER_SCHEMAS?: string;
65+
OS_SKIP_SCHEMA_SYNC?: string;
6566

6667
// — Storage (S3/R2) —
6768
OS_STORAGE_ADAPTER?: string;
@@ -143,6 +144,7 @@ const FORWARDED_ENV_KEYS: readonly (keyof Env)[] = [
143144
'OS_DATA_DIR',
144145
'OS_PROVISION_SYNC',
145146
'OS_EAGER_SCHEMAS',
147+
'OS_SKIP_SCHEMA_SYNC',
146148
// storage
147149
'OS_STORAGE_ADAPTER',
148150
'OS_STORAGE_LOCAL_DIR',
@@ -279,6 +281,17 @@ export class CloudContainer extends Container<Env> {
279281
OS_KERNEL_CACHE_SIZE: '50',
280282
OS_KERNEL_TTL_MS: '1800000',
281283
OS_ENV_CACHE_TTL_MS: '300000',
284+
// Cold-start optimization: schema sync (one round-trip per
285+
// sys_* table on a remote Postgres) routinely runs ~30–60s
286+
// against a cold Neon DB, which exceeds Cloudflare Workers'
287+
// inbound-request budget (~30s). The container can never
288+
// finish booting on a fresh request because the platform
289+
// tears down the in-flight DO invocation when the inbound
290+
// request expires. Move DDL out-of-band: run
291+
// `pnpm --filter @objectstack/cloud migrate` against the
292+
// production DB before deploying the image, then let the
293+
// container assume the schema is already there.
294+
OS_SKIP_SCHEMA_SYNC: '1',
282295
// Public URL the better-auth instance issues redirects from. MUST
283296
// match the origin the browser hits, otherwise sign-up / OAuth
284297
// callbacks fail with "Invalid origin". Override per environment

apps/cloud/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"typecheck": "tsc --noEmit",
1313
"test": "objectstack test",
1414
"test:production-flow": "tsx test/production-flow.test.ts",
15+
"migrate": "tsx scripts/migrate.ts",
1516
"cf:build": "bash scripts/deploy-cloudflare.sh --skip-push --skip-deploy",
1617
"cf:push": "bash scripts/deploy-cloudflare.sh --skip-build --skip-deploy",
1718
"cf:deploy": "bash scripts/deploy-cloudflare.sh",

apps/cloud/scripts/deploy-cloudflare.sh

Lines changed: 55 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,18 @@ fi
2424
: "${CF_IMAGE_TAG:=$(cd "$REPO_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo latest)}"
2525
: "${CF_PLATFORM:=linux/amd64}"
2626

27-
SKIP_BUILD=0; SKIP_PUSH=0; SKIP_DEPLOY=0; DRY_RUN=0
27+
SKIP_BUILD=0; SKIP_PUSH=0; SKIP_DEPLOY=0; SKIP_MIGRATE=0; DRY_RUN=0
2828
while [[ $# -gt 0 ]]; do
2929
case "$1" in
3030
--) shift ;;
31-
--skip-build) SKIP_BUILD=1; shift ;;
32-
--skip-push) SKIP_PUSH=1; shift ;;
33-
--skip-deploy) SKIP_DEPLOY=1; shift ;;
34-
--dry-run) DRY_RUN=1; shift ;;
35-
--tag) CF_IMAGE_TAG="$2"; shift 2 ;;
36-
--tag=*) CF_IMAGE_TAG="${1#--tag=}"; shift ;;
37-
-h|--help) grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'; exit 0 ;;
31+
--skip-build) SKIP_BUILD=1; shift ;;
32+
--skip-push) SKIP_PUSH=1; shift ;;
33+
--skip-deploy) SKIP_DEPLOY=1; shift ;;
34+
--skip-migrate) SKIP_MIGRATE=1; shift ;;
35+
--dry-run) DRY_RUN=1; shift ;;
36+
--tag) CF_IMAGE_TAG="$2"; shift 2 ;;
37+
--tag=*) CF_IMAGE_TAG="${1#--tag=}"; shift ;;
38+
-h|--help) grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'; exit 0 ;;
3839
*) echo "unknown arg: $1" >&2; exit 2 ;;
3940
esac
4041
done
@@ -81,7 +82,7 @@ run() { if [[ $DRY_RUN -eq 1 ]]; then echo "[dry-run] $*"; else "$@"; fi; }
8182

8283
if [[ $SKIP_BUILD -eq 0 ]]; then
8384
echo ""
84-
echo "▶ [1/3] docker buildx build"
85+
echo "▶ [1/4] docker buildx build"
8586
command -v docker >/dev/null || { echo "✗ docker not installed" >&2; exit 1; }
8687
run docker buildx build \
8788
--platform "$CF_PLATFORM" \
@@ -92,12 +93,52 @@ if [[ $SKIP_BUILD -eq 0 ]]; then
9293
--load \
9394
"$REPO_ROOT"
9495
else
95-
echo "▶ [1/3] skipped (--skip-build)"
96+
echo "▶ [1/4] skipped (--skip-build)"
97+
fi
98+
99+
# ─────────────────────────────────────────────────────────────────────
100+
# Migration step (out-of-band schema sync against the production DB).
101+
#
102+
# Runs the kernel locally with OS_MIGRATE_AND_EXIT=1 so the
103+
# ObjectQLPlugin.start() schema sync runs ONCE against Neon/Turso
104+
# from the deploy machine (which has no 30s wallclock budget). The
105+
# container then ships with OS_SKIP_SCHEMA_SYNC=1 baked in (see
106+
# cloudflare/worker.ts) and skips DDL on every cold boot.
107+
#
108+
# Without this, the container is killed mid-DDL on every cold start
109+
# because Cloudflare Workers' inbound-request budget (~30s) is shorter
110+
# than a fresh remote-DB schema sync (~30–60s for ~30 sys_* tables).
111+
#
112+
# Requires the prebuilt config (apps/cloud/dist/objectstack.config.js)
113+
# — the build step above produces it inside the Docker image, so we
114+
# trigger a host-side build if needed.
115+
# ─────────────────────────────────────────────────────────────────────
116+
if [[ $SKIP_MIGRATE -eq 0 && $SKIP_PUSH -eq 0 ]]; then
117+
echo ""
118+
echo "▶ [2/4] schema migration (OS_SKIP_SCHEMA_SYNC=0, OS_MIGRATE_AND_EXIT=1)"
119+
if [[ ! -f "$APP_DIR/dist/objectstack.config.js" ]]; then
120+
echo " · dist/ not found — building host-side first"
121+
run pnpm --dir "$APP_DIR" build
122+
fi
123+
if [[ $DRY_RUN -eq 1 ]]; then
124+
echo "[dry-run] pnpm --dir $APP_DIR migrate"
125+
else
126+
if ! pnpm --dir "$APP_DIR" migrate; then
127+
echo "" >&2
128+
echo "✗ schema migration failed." >&2
129+
echo " Check OS_DATABASE_URL in $APP_DIR/.env.cloudflare.secrets and" >&2
130+
echo " re-run \`pnpm --dir $APP_DIR migrate\` with DEBUG=* for details." >&2
131+
echo " Pass --skip-migrate if you've already migrated separately." >&2
132+
exit 1
133+
fi
134+
fi
135+
else
136+
echo "▶ [2/4] skipped ($([[ $SKIP_MIGRATE -eq 1 ]] && echo --skip-migrate || echo --skip-push))"
96137
fi
97138

98139
if [[ $SKIP_PUSH -eq 0 ]]; then
99140
echo ""
100-
echo "▶ [2/3] wrangler containers push"
141+
echo "▶ [3/4] wrangler containers push"
101142
if [[ $DRY_RUN -eq 1 ]]; then
102143
echo "[dry-run] npx --yes wrangler containers push $IMAGE"
103144
else
@@ -116,12 +157,12 @@ if [[ $SKIP_PUSH -eq 0 ]]; then
116157
fi
117158
fi
118159
else
119-
echo "▶ [2/3] skipped (--skip-push)"
160+
echo "▶ [3/4] skipped (--skip-push)"
120161
fi
121162

122163
if [[ $SKIP_DEPLOY -eq 0 ]]; then
123164
echo ""
124-
echo "▶ [3/3] update wrangler.toml image → $IMAGE"
165+
echo "▶ [4/4] update wrangler.toml image → $IMAGE"
125166
if [[ $DRY_RUN -eq 0 ]]; then
126167
if [[ "$OSTYPE" == "darwin"* ]]; then
127168
sed -i '' -E "s|^image = \".*\"|image = \"$IMAGE\"|" "$WRANGLER_TOML"
@@ -133,7 +174,7 @@ if [[ $SKIP_DEPLOY -eq 0 ]]; then
133174
echo "▶ wrangler deploy"
134175
run npx --yes wrangler deploy --config "$WRANGLER_TOML"
135176
else
136-
echo "▶ [3/3] skipped (--skip-deploy)"
177+
echo "▶ [4/4] skipped (--skip-deploy)"
137178
fi
138179

139180
echo ""

apps/cloud/scripts/migrate.ts

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
2+
3+
/**
4+
* Out-of-band schema migration for ObjectStack Cloud.
5+
*
6+
* Why this exists
7+
* ───────────────
8+
* The Cloudflare Containers runtime gives an inbound Worker request
9+
* roughly 30s of wallclock before the platform tears the DO invocation
10+
* down. A cold control-plane boot against a fresh Neon DB has to:
11+
*
12+
* 1. Open a Postgres connection (1–3s cold).
13+
* 2. Run `CREATE TABLE IF NOT EXISTS` for every `sys_*` object —
14+
* one round-trip per table because `driver-sql` does not yet
15+
* implement `batchSchemaSync` (verified at
16+
* packages/plugins/driver-sql/src/sql-driver.ts:108).
17+
* 3. Hydrate `sys_metadata`, then DDL any custom tables that just
18+
* came in (Phase 3 in `ObjectQLPlugin.start`).
19+
*
20+
* Steps 1+2 alone routinely take 30–60s, so the container is killed
21+
* mid-DDL on every cold request and never reaches `listen(4000)`. The
22+
* three timeouts we previously bumped (`startupTimeout`,
23+
* `portReadyTimeoutMS`, `instanceGetTimeoutMS`) are necessary but
24+
* insufficient — the platform's request-budget is the actual wall.
25+
*
26+
* Strategy
27+
* ────────
28+
* Run schema sync ONCE from the deploy machine against the production
29+
* DB, then ship the container with `OS_SKIP_SCHEMA_SYNC=1` so cold
30+
* boots only do connection-open + sys_metadata hydration (~sub-second
31+
* on warm Neon).
32+
*
33+
* How it works
34+
* ────────────
35+
* This script just delegates to the existing `objectstack serve`
36+
* machinery (which already knows how to load `dist/objectstack.config.js`,
37+
* register every plugin, and bootstrap the kernel) but with two env
38+
* overrides:
39+
*
40+
* • `OS_SKIP_SCHEMA_SYNC=0` — force `ObjectQLPlugin.start()` to
41+
* actually run `syncRegisteredSchemas()` even if the operator's
42+
* shell exports the production default.
43+
* • `OS_MIGRATE_AND_EXIT=1` — `serve.ts` watches for this and
44+
* `kernel.shutdown() + process.exit(0)` immediately after
45+
* `runtime.start()` resolves successfully, instead of holding the
46+
* port open.
47+
*
48+
* The `OS_DATABASE_URL` (and any other secrets) must be present in
49+
* the script's env when it runs — we do NOT push secrets here, they
50+
* come from `apps/cloud/.env.cloudflare.secrets` (loaded by the
51+
* caller, e.g. `deploy-cloudflare.sh`) or from the operator's shell.
52+
*
53+
* Usage
54+
* ─────
55+
* pnpm --filter @objectstack/cloud build # produces dist/objectstack.config.js
56+
* pnpm --filter @objectstack/cloud migrate # this script
57+
*
58+
* Or, automatically as part of `cf:deploy` (see deploy-cloudflare.sh).
59+
*/
60+
61+
import { spawn } from 'node:child_process';
62+
import path from 'node:path';
63+
import { fileURLToPath } from 'node:url';
64+
import { existsSync, readFileSync } from 'node:fs';
65+
66+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
67+
const APP_DIR = path.resolve(__dirname, '..');
68+
const CONFIG_PATH = path.join(APP_DIR, 'dist', 'objectstack.config.js');
69+
const SECRETS_FILE = path.join(APP_DIR, '.env.cloudflare.secrets');
70+
71+
// ── 1. Verify the prebuilt config exists ─────────────────────────────
72+
if (!existsSync(CONFIG_PATH)) {
73+
console.error(`✗ ${CONFIG_PATH} not found.`);
74+
console.error(' Run `pnpm --filter @objectstack/cloud build` first.');
75+
process.exit(1);
76+
}
77+
78+
// ── 2. Load .env.cloudflare.secrets if present ───────────────────────
79+
// Mirrors the parser in setup-cloudflare-secrets.sh — values may
80+
// contain `&`, `;`, `$`, etc., so we MUST NOT use `bash source`-style
81+
// parsing. We strip ONE optional layer of surrounding `'` or `"`.
82+
function loadEnvFile(file: string): Record<string, string> {
83+
if (!existsSync(file)) return {};
84+
const out: Record<string, string> = {};
85+
const text = readFileSync(file, 'utf8');
86+
for (const rawLine of text.split(/\r?\n/)) {
87+
const line = rawLine.trim();
88+
if (!line || line.startsWith('#')) continue;
89+
const m = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
90+
if (!m) continue;
91+
const key = m[1];
92+
let value = m[2];
93+
if (
94+
(value.startsWith('"') && value.endsWith('"')) ||
95+
(value.startsWith("'") && value.endsWith("'"))
96+
) {
97+
value = value.slice(1, -1);
98+
}
99+
out[key] = value;
100+
}
101+
return out;
102+
}
103+
104+
const fileEnv = loadEnvFile(SECRETS_FILE);
105+
// Don't clobber values the operator already set in their shell — the
106+
// shell wins because that's where one-off overrides happen.
107+
const mergedEnv: NodeJS.ProcessEnv = { ...fileEnv, ...process.env };
108+
109+
// ── 3. Sanity-check that we have a real DB URL ───────────────────────
110+
const dbUrl = mergedEnv.OS_DATABASE_URL || mergedEnv.OS_CONTROL_DATABASE_URL;
111+
if (!dbUrl) {
112+
console.error('✗ OS_DATABASE_URL is not set.');
113+
console.error(` Set it in ${SECRETS_FILE} or export it in your shell.`);
114+
process.exit(1);
115+
}
116+
if (dbUrl.startsWith('file:') || dbUrl.includes(':memory:')) {
117+
console.error(`✗ OS_DATABASE_URL points at a local file (${dbUrl}).`);
118+
console.error(' Migrating local SQLite is pointless — the container ships');
119+
console.error(' with a fresh ephemeral filesystem. Point at production Neon/Turso.');
120+
process.exit(1);
121+
}
122+
123+
// ── 4. Force schema sync ON, exit-after-bootstrap ON ─────────────────
124+
mergedEnv.OS_SKIP_SCHEMA_SYNC = '0';
125+
mergedEnv.OS_MIGRATE_AND_EXIT = '1';
126+
// Run on a non-conflicting port so we don't fight a `pnpm dev` instance.
127+
mergedEnv.PORT = mergedEnv.MIGRATE_PORT ?? '4099';
128+
// Quiet down the runtime banner — the migration banner is what matters here.
129+
mergedEnv.OS_DISABLE_CONSOLE = '1';
130+
131+
const redactedUrl = dbUrl.replace(/:\/\/[^@]+@/, '://***@');
132+
console.log('────────────────────────────────────────────────────────────');
133+
console.log(' ObjectStack Cloud — out-of-band schema migration');
134+
console.log('────────────────────────────────────────────────────────────');
135+
console.log(` Config : ${path.relative(process.cwd(), CONFIG_PATH)}`);
136+
console.log(` Target : ${redactedUrl}`);
137+
console.log(' Mode : OS_MIGRATE_AND_EXIT=1, OS_SKIP_SCHEMA_SYNC=0');
138+
console.log('────────────────────────────────────────────────────────────');
139+
140+
// ── 5. Delegate to `objectstack serve --prebuilt` ────────────────────
141+
// We use the local CLI binary from the workspace so this works inside
142+
// `pnpm --filter @objectstack/cloud migrate` without a separate npx
143+
// resolution. `--prebuilt` skips esbuild/bundle-require — the dist file
144+
// is already pure ESM.
145+
const cliBin = path.join(APP_DIR, 'node_modules', '.bin', 'objectstack');
146+
// `--no-server` skips the Hono HTTP server plugin entirely so we don't
147+
// have to bind a port at all. Schema sync lives in
148+
// `ObjectQLPlugin.start()`, which runs regardless. `--no-ui` skips the
149+
// Studio static asset plugin (irrelevant for migration).
150+
const args = ['serve', CONFIG_PATH, '--prebuilt', '--no-ui', '--no-server'];
151+
152+
const child = spawn(cliBin, args, {
153+
cwd: APP_DIR,
154+
env: mergedEnv,
155+
stdio: 'inherit',
156+
});
157+
158+
child.on('exit', (code, signal) => {
159+
if (signal) {
160+
console.error(`✗ migrate killed by signal ${signal}`);
161+
process.exit(1);
162+
}
163+
process.exit(code ?? 0);
164+
});
165+
child.on('error', (err) => {
166+
console.error(`✗ failed to spawn objectstack CLI: ${err.message}`);
167+
process.exit(1);
168+
});

apps/cloud/wrangler.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ compatibility_flags = ["nodejs_compat"]
4545
# rebuild + re-push to ship a new version, then run `wrangler deploy`.
4646
[[containers]]
4747
class_name = "CloudContainer"
48-
image = "registry.cloudflare.com/2846eb40a60f4738e292b90dcd8cce10/objectstack-cloud:acd4efe6"
48+
image = "registry.cloudflare.com/2846eb40a60f4738e292b90dcd8cce10/objectstack-cloud:9d417fef"
4949
max_instances = 3
5050
instance_type = "standard-1"
5151

packages/cli/src/commands/serve.ts

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -976,6 +976,24 @@ export default class Serve extends Command {
976976
await new Promise(r => setTimeout(r, 100));
977977
restoreOutput();
978978

979+
// ── Migrate-and-exit short-circuit ─────────────────────────────
980+
// Out-of-band migration mode: the caller (e.g.
981+
// `apps/cloud/scripts/migrate.ts`) just wants the kernel
982+
// bootstrap (ObjectQLPlugin → schema sync → metadata hydration)
983+
// to run once against the configured database, then exit. The
984+
// HTTP server has already bound `port` at this point but we
985+
// never accept a request — shutdown immediately so the deploy
986+
// pipeline can move on.
987+
if (process.env.OS_MIGRATE_AND_EXIT === '1') {
988+
console.log(chalk.green(`✓ Migration complete (${loadedPlugins.length} plugins started against ${redactDbUrl(resolvedDatabaseUrl) || 'configured DB'})`));
989+
try {
990+
await kernel.shutdown();
991+
} catch (err: any) {
992+
console.warn(chalk.yellow(` ⚠ shutdown warning: ${err?.message ?? err}`));
993+
}
994+
process.exit(0);
995+
}
996+
979997
// ── Driver introspection ──────────────────────────────────────
980998
// When the driver was registered by an app preset / per-project
981999
// factory (ProjectKernelFactory) instead of serve.ts's own

0 commit comments

Comments
 (0)