Merge pull request #357 from cipherstash/encryption-migrations

feat(cli, migrate): add `stash encrypt` commands + @cipherstash/migrate

Author: calvinbrewer

.changeset/encryption-migrations.md

@@ -0,0 +1,18 @@
+---
+'stash': minor
+'@cipherstash/migrate': minor
+---
+
+Add `stash encrypt` command group and `@cipherstash/migrate` library for plaintext → encrypted column migrations.
+
+New CLI commands:
+
+- `stash encrypt status` — per-column migration status (phase, backfill progress, drift between intent and state, EQL registration).
+- `stash encrypt plan` — diff `.cipherstash/migrations.json` (intent) vs observed state.
+- `stash encrypt backfill --table <t> --column <c>` — resumable, idempotent, chunked encryption of plaintext into `<col>_encrypted`. Uses the user's encryption client (Protect/Stack). SIGINT-safe; re-run to resume. The first run on a column prompts to confirm dual-writes are deployed (or accept `--confirm-dual-writes-deployed` for non-interactive contexts), records the `dual_writing` transition in `cs_migrations`, then runs the chunked encryption loop. `--force` re-encrypts every plaintext row regardless of current state — recovery path for drift caused by an earlier backfill running before dual-writes were actually live.
+- `stash encrypt cutover --table <t> --column <c>` — runs `eql_v2.rename_encrypted_columns()` inside a transaction; optionally forces Proxy config refresh via `CIPHERSTASH_PROXY_URL`. After cutover, apps reading `<col>` transparently receive the encrypted column.
+- `stash encrypt drop --table <t> --column <c>` — generates a migration file that drops the old plaintext column.
+
+`stash db install` now also installs a `cipherstash.cs_migrations` table used to track per-column migration runtime state (current phase, backfill cursor, rows processed). The table is append-only (event-log shape) and kept separate from `eql_v2_configuration` which remains the authoritative EQL intent store used by Proxy.
+
+The new `@cipherstash/migrate` package exposes the same primitives as a library for users who want to embed backfill in their own workers or cron jobs — all commands are thin wrappers around its exports (`runBackfill`, `appendEvent`, `latestByColumn`, `progress`, `renameEncryptedColumns`, `reloadConfig`, `readManifest`, `writeManifest`).

docs/plans/encryption-migrations.md

@@ -41,6 +41,7 @@
 	},
 	"dependencies": {
 		"@cipherstash/auth": "catalog:repo",
+		"@cipherstash/migrate": "workspace:*",
 		"@clack/prompts": "0.10.1",
 		"dotenv": "16.4.7",
 		"jiti": "2.6.1",

packages/cli/package.json

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# End-to-end smoke test for `stash encrypt`.
+#
+# Requires a local Postgres you have superuser on. Creates & destroys
+# `stash_e2e_test`. Requires CipherStash credentials in the environment
+# for the actual encryption step (CS_CLIENT_ACCESS_KEY etc).
+#
+# Usage: bash packages/cli/scripts/e2e-encrypt.sh
+
+set -euo pipefail
+
+DB=${STASH_E2E_DB:-stash_e2e_test}
+HOST=${STASH_E2E_HOST:-localhost}
+DATABASE_URL="postgres://${USER}@${HOST}/${DB}"
+STASH="$(cd "$(dirname "$0")/../dist/bin" && pwd)/stash.js"
+FIXTURES="$(cd "$(dirname "$0")/fixtures" && pwd)"
+
+if [ ! -x "$STASH" ]; then
+  echo "CLI not built. Run: pnpm --filter @cipherstash/cli build" >&2
+  exit 1
+fi
+
+psql -h "$HOST" -d postgres -c "DROP DATABASE IF EXISTS ${DB}" >/dev/null
+psql -h "$HOST" -d postgres -c "CREATE DATABASE ${DB}" >/dev/null
+
+export DATABASE_URL
+
+echo "==> 1. Install EQL + cs_migrations"
+"$STASH" db install --force
+
+echo "==> 2. Seed 5000 plaintext users"
+psql "$DATABASE_URL" -f "$FIXTURES/seed-users.sql" >/dev/null
+psql "$DATABASE_URL" -c "ALTER TABLE users ADD COLUMN email_encrypted eql_v2_encrypted" >/dev/null
+
+echo "==> 3. Backfill with interrupt/resume (dual-writes confirmed via flag for non-interactive run)"
+"$STASH" encrypt backfill --table users --column email --chunk-size 500 --confirm-dual-writes-deployed &
+PID=$!
+sleep 2
+kill -INT "$PID" || true
+wait "$PID" || true
+"$STASH" encrypt backfill --table users --column email
+
+REMAINING=$(psql "$DATABASE_URL" -At -c "SELECT count(*) FROM users WHERE email_encrypted IS NULL")
+if [ "$REMAINING" != "0" ]; then
+  echo "FAIL: ${REMAINING} rows still unencrypted" >&2
+  exit 1
+fi
+echo "OK: all 5000 rows encrypted"
+
+echo "==> 4. Status"
+"$STASH" encrypt status
+
+echo "==> 5. Cutover"
+"$STASH" encrypt cutover --table users --column email
+
+echo "==> 6. Drop"
+"$STASH" encrypt drop --table users --column email --migrations-dir "$(pwd)/drizzle"
+
+echo "==> Done."

packages/cli/scripts/e2e-encrypt.sh

@@ -0,0 +1,16 @@
+-- Seed a users table with plaintext emails for e2e backfill testing.
+--
+-- The encrypted target column must be created separately (drizzle-kit /
+-- stash db push route), after which the backfill encrypts `email` → `email_encrypted`.
+
+DROP TABLE IF EXISTS users CASCADE;
+
+CREATE TABLE users (
+  id        bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+  email     text NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now()
+);
+
+INSERT INTO users (email)
+SELECT 'user-' || g || '@example.com'
+FROM generate_series(1, 5000) AS g;

packages/cli/scripts/fixtures/seed-users.sql

@@ -79,14 +79,21 @@ Commands:
 
   db install           Scaffold stash.config.ts (if missing) and install EQL extensions
   db upgrade           Upgrade EQL extensions to the latest version
-  db push              Push encryption schema to database (CipherStash Proxy only)
+  db push              Push encryption schema (writes pending if active config already exists)
+  db activate          Promote pending → active without renames (use after additive db push)
   db validate          Validate encryption schema
   db migrate           Run pending encrypt config migrations
   db status            Show EQL installation status
   db test-connection   Test database connectivity
 
   schema build         Build an encryption schema from your database
 
+  encrypt status       Show per-column migration status (phase, progress, drift)
+  encrypt plan         Diff intent (.cipherstash/migrations.json) vs observed state
+  encrypt backfill     Resumably encrypt plaintext into the encrypted column
+  encrypt cutover      Rename swap encrypted → primary column
+  encrypt drop         Generate a migration to drop the plaintext column
+
   env                  (experimental) Print production env vars for deployment
 
 Options:
@@ -198,6 +205,13 @@ async function runDbCommand(
       await pushCommand({ dryRun: flags['dry-run'], databaseUrl })
       break
     }
+    case 'activate': {
+      const { activateCommand } = await requireStack(
+        () => import('../commands/db/activate.js'),
+      )
+      await activateCommand({ databaseUrl })
+      break
+    }
     case 'validate': {
       const { validateCommand } = await requireStack(
         () => import('../commands/db/validate.js'),
@@ -226,6 +240,90 @@ async function runDbCommand(
   }
 }
 
+async function runEncryptCommand(
+  sub: string | undefined,
+  flags: Record<string, boolean>,
+  values: Record<string, string>,
+) {
+  switch (sub) {
+    case 'status': {
+      const { statusCommand } = await requireStack(
+        () => import('../commands/encrypt/status.js'),
+      )
+      await statusCommand()
+      break
+    }
+    case 'plan': {
+      const { planCommand } = await requireStack(
+        () => import('../commands/encrypt/plan.js'),
+      )
+      await planCommand()
+      break
+    }
+    case 'backfill': {
+      const table = requireValue(values, 'table')
+      const column = requireValue(values, 'column')
+      const { backfillCommand } = await requireStack(
+        () => import('../commands/encrypt/backfill.js'),
+      )
+      await backfillCommand({
+        table,
+        column,
+        pkColumn: values['pk-column'],
+        chunkSize: values['chunk-size']
+          ? Number(values['chunk-size'])
+          : undefined,
+        encryptedColumn: values['encrypted-column'],
+        schemaColumnKey: values['schema-column-key'],
+        confirmDualWritesDeployed: flags['confirm-dual-writes-deployed'],
+        force: flags.force,
+      })
+      break
+    }
+    case 'cutover': {
+      const table = requireValue(values, 'table')
+      const column = requireValue(values, 'column')
+      const { cutoverCommand } = await requireStack(
+        () => import('../commands/encrypt/cutover.js'),
+      )
+      await cutoverCommand({
+        table,
+        column,
+        proxyUrl: values['proxy-url'],
+        migrationsDir: values['migrations-dir'],
+      })
+      break
+    }
+    case 'drop': {
+      const table = requireValue(values, 'table')
+      const column = requireValue(values, 'column')
+      const { dropCommand } = await requireStack(
+        () => import('../commands/encrypt/drop.js'),
+      )
+      await dropCommand({
+        table,
+        column,
+        migrationsDir: values['migrations-dir'],
+      })
+      break
+    }
+    default:
+      p.log.error(`Unknown encrypt subcommand: ${sub ?? '(none)'}`)
+      console.log()
+      console.log(HELP)
+      process.exit(1)
+  }
+}
+
+function requireValue(values: Record<string, string>, key: string): string {
+  const v = values[key]
+  if (!v) {
+    p.log.error(`Missing required --${key} value.`)
+    process.exit(1)
+  }
+  return v
+}
+
 async function runSchemaCommand(
   sub: string | undefined,
   flags: Record<string, boolean>,
@@ -277,6 +375,9 @@ async function main() {
     case 'db':
       await runDbCommand(subcommand, flags, values)
       break
+    case 'encrypt':
+      await runEncryptCommand(subcommand, flags, values)
+      break
     case 'schema':
       await runSchemaCommand(subcommand, flags, values)
       break

packages/cli/src/bin/stash.ts

@@ -0,0 +1,73 @@
+import { detectPackageManager, runnerCommand } from '@/commands/init/utils.js'
+import { loadStashConfig } from '@/config/index.js'
+import { activateConfig, migrateConfig } from '@cipherstash/migrate'
+import * as p from '@clack/prompts'
+import pg from 'pg'
+
+/**
+ * `stash db activate` — promote the pending EQL configuration to active
+ * **without** renaming any columns.
+ *
+ * Used after `stash db push` when the new config is purely additive
+ * (e.g. registering a brand-new encrypted column on a project that
+ * already has an active config) — there are no `<col>_encrypted` twins
+ * to rename, so the cut-over rename step is unnecessary.
+ *
+ * For path 3 (existing populated column → encrypted via lifecycle), use
+ * `stash encrypt cutover` instead. Cutover does the same activation but
+ * also runs the physical rename.
+ *
+ * Mechanics: chains `eql_v2.migrate_config()` (pending → encrypting) and
+ * `eql_v2.activate_config()` (encrypting → active) inside a single
+ * transaction. Errors out clearly when there is no pending config to
+ * activate.
+ */
+export interface ActivateCommandOptions {
+  databaseUrl?: string
+}
+
+export async function activateCommand(
+  options: ActivateCommandOptions,
+): Promise<void> {
+  p.intro(runnerCommand(detectPackageManager(), 'stash db activate'))
+
+  const stashConfig = await loadStashConfig({
+    databaseUrlFlag: options.databaseUrl,
+  })
+  const client = new pg.Client({ connectionString: stashConfig.databaseUrl })
+  let exitCode = 0
+
+  try {
+    await client.connect()
+
+    const pending = await client.query<{ exists: boolean }>(
+      "SELECT EXISTS(SELECT 1 FROM public.eql_v2_configuration WHERE state = 'pending') AS exists",
+    )
+    if (pending.rows[0]?.exists !== true) {
+      p.log.error(
+        'No pending EQL configuration to activate. Run `stash db push` first to register a change.',
+      )
+      exitCode = 1
+      return
+    }
+
+    await client.query('BEGIN')
+    try {
+      await migrateConfig(client)
+      await activateConfig(client)
+      await client.query('COMMIT')
+    } catch (err) {
+      await client.query('ROLLBACK').catch(() => {})
+      throw err
+    }
+
+    p.log.success('Pending configuration promoted to active.')
+    p.outro('Done.')
+  } catch (error) {
+    p.log.error(error instanceof Error ? error.message : 'Activation failed.')
+    exitCode = 1
+  } finally {
+    await client.end()
+  }
+  if (exitCode) process.exit(exitCode)
+}