Skip to content

Commit ecb0d8a

Browse files
committed
feat(migration): implement state file migration to hash store
This commit introduces a new migration process that transforms legacy state files from the `.vapi-state.<org>.json` format to a slimmed structure, mapping resource names to their UUIDs. The migration also seeds a new per-developer hash store located at `.vapi-state-hash/<org>/<uuid>`, which holds the last known platform state hashes for each resource. Key changes include: - Addition of `migrate-cmd.ts` for executing the migration. - Implementation of `migrate-hash-store.ts` to handle the migration logic. - New `hash-store.ts` module to manage the hash store operations. - Updates to existing commands (`pull`, `push`, `apply`) to ensure they validate state files against the new format and utilize the hash store for drift detection. This migration is idempotent and does not require a VAPI_TOKEN, allowing it to run across all organizations without configuration. The changes enhance the drift detection mechanism by separating state management from hash storage, improving clarity and reliability in resource synchronization.
1 parent 79f4008 commit ecb0d8a

13 files changed

Lines changed: 452 additions & 127 deletions

.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ tmp/
2727
# Operator-local; not shared.
2828
.vapi-state.*.snapshots/
2929

30+
# Per-developer drift baseline store (.vapi-state-hash/<org>/<uuid>).
31+
# Holds "the last platform state I saw" per resource. Local, not shared —
32+
# committing it would make one developer's baseline everyone's.
33+
.vapi-state-hash/
34+
3035
# Local agent state
3136
.claude/
3237
.agent/

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"apply": "tsx src/apply-cmd.ts",
1111
"push": "tsx src/push-cmd.ts",
1212
"pull": "tsx src/pull-cmd.ts",
13+
"migrate": "tsx src/migrate-cmd.ts",
1314
"call": "bash -c 'exec tsx src/call-cmd.ts \"$@\" 2> >(grep --line-buffered -v \"buffer underflow\" >&2)' --",
1415
"cleanup": "tsx src/cleanup-cmd.ts",
1516
"validate": "tsx src/validate-cmd.ts",

src/apply.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { execSync } from "child_process";
22
import { dirname, join, resolve } from "path";
33
import { fileURLToPath } from "url";
4+
import { assertStateMigrated } from "./migrate-hash-store.ts";
45

56
// ─────────────────────────────────────────────────────────────────────────────
67
// Apply: Pull → Merge → Push (safe bidirectional sync)
@@ -29,6 +30,12 @@ export async function runApply(): Promise<void> {
2930
const allArgs = process.argv.slice(3);
3031
const hasForce = allArgs.includes("--force");
3132

33+
// Fail fast with one clear message if this org's state file is still legacy,
34+
// rather than letting the spawned pull/push subprocess surface it later.
35+
if (env && SLUG_RE.test(env)) {
36+
assertStateMigrated(join(BASE_DIR, `.vapi-state.${env}.json`));
37+
}
38+
3239
// Apply's job is to push local up; default pull drift resolution to "ours"
3340
// unless the operator explicitly passes --resolve=theirs|fail (CI).
3441
const pullArgsList = allArgs.filter((a) => a !== "--force");

src/audit.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@
2121

2222
import { readFile } from "fs/promises";
2323
import { join } from "path";
24-
import { matchesIgnore, RESOURCES_DIR } from "./config.ts";
24+
import { matchesIgnore, RESOURCES_DIR, VAPI_ENV } from "./config.ts";
2525
import { credentialReverseMap } from "./credentials.ts";
2626
import { classifyDrift } from "./drift.ts";
27+
import { readBaseline } from "./hash-store.ts";
2728
import { findOrphanResourceIds } from "./new-file-gate.ts";
2829
import { canonicalizeForHash, type VapiResource } from "./canonical.ts";
2930
import { fetchAllResources, listExistingResourceIds } from "./pull.ts";
@@ -245,7 +246,7 @@ function checkContentIdentical(
245246
): { findings: AuditFinding[]; identicalSlugs: Set<string> } {
246247
const byHash = new Map<string, string[]>();
247248
for (const [resourceId, entry] of Object.entries(state[type])) {
248-
const hash = entry.lastPulledHash;
249+
const hash = readBaseline(VAPI_ENV, entry.uuid);
249250
if (!hash) continue;
250251
const slugs = byHash.get(hash) ?? [];
251252
slugs.push(resourceId);
@@ -383,13 +384,13 @@ function checkContentDrift(
383384
if (!remoteResource) continue;
384385

385386
// canonicalizeForHash centralizes the 3-step pipeline + _platformDefault
386-
// mutation so the hash agrees with pull-write's lastPulledHash basis.
387+
// mutation so the hash agrees with the pull-write baseline basis.
387388
const platformHash = hashPayload(
388389
canonicalizeForHash(remoteResource, state, credReverse),
389390
);
390391
const direction = classifyDrift({
391392
localHash,
392-
lastPulledHash: entry.lastPulledHash,
393+
lastPulledHash: readBaseline(VAPI_ENV, entry.uuid),
393394
platformHash,
394395
});
395396
if (direction === "clean") continue;

src/drift.ts

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import { vapiGet, VapiApiError } from "./api.ts";
2222
import { canonicalizeForHash, type VapiResource } from "./canonical.ts";
2323
import { credentialReverseMap } from "./credentials.ts";
24+
import { readBaseline } from "./hash-store.ts";
2425
import { hashLocalResource } from "./resources.ts";
2526
import { hashPayload } from "./state-serialize.ts";
2627
import type { ResourceType, StateFile } from "./types.ts";
@@ -93,19 +94,30 @@ export async function checkDriftForUpdate(options: {
9394
resourceType: ResourceType;
9495
resourceId: string; // local resource id
9596
state: StateFile;
97+
env: string; // org slug — keys the hash store
9698
overwrite: boolean;
9799
}): Promise<DriftCheckResult> {
98-
const { endpoint, resourceLabel, resourceType, resourceId, state, overwrite } =
99-
options;
100-
100+
const {
101+
endpoint,
102+
resourceLabel,
103+
resourceType,
104+
resourceId,
105+
state,
106+
env,
107+
overwrite,
108+
} = options;
109+
110+
// The drift baseline ("last platform state I saw") lives in the hash store,
111+
// keyed by the resource's UUID, not in the state file anymore.
101112
const entry = state[resourceType]?.[resourceId];
102-
if (!entry?.lastPulledHash) {
113+
const baseline = entry ? readBaseline(env, entry.uuid) : undefined;
114+
if (!baseline) {
103115
return {
104116
ok: true,
105117
reason: "no-baseline",
106118
message:
107119
` ⚠️ drift check skipped for ${resourceLabel} ${resourceId}: ` +
108-
`no lastPulledHash in state. Run \`npm run pull\` to establish a baseline.`,
120+
`no baseline in .vapi-state-hash. Run \`npm run pull\` to establish one.`,
109121
};
110122
}
111123

@@ -125,15 +137,15 @@ export async function checkDriftForUpdate(options: {
125137
const platformHash = hashPayload(
126138
canonicalizeForHash(remote as VapiResource, state, credReverse),
127139
);
128-
if (platformHash === entry.lastPulledHash) {
140+
if (platformHash === baseline) {
129141
return { ok: true, reason: "match", platformHash };
130142
}
131143

132-
// On-disk hash in the same basis as `lastPulledHash`. Absent a local file
144+
// On-disk hash in the same basis as the baseline. Absent a local file
133145
// (rare on an update path), fall back to the baseline so the direction is
134146
// dashboard-ahead rather than a phantom both-diverged.
135147
const localHash =
136-
hashLocalResource(resourceType, resourceId) ?? entry.lastPulledHash;
148+
hashLocalResource(resourceType, resourceId) ?? baseline;
137149

138150
// Local and platform are byte-identical → there is nothing to reconcile and
139151
// the PATCH is a no-op. NEVER block here, even if `lastPulledHash` disagrees
@@ -150,7 +162,7 @@ export async function checkDriftForUpdate(options: {
150162

151163
const direction = classifyDrift({
152164
localHash,
153-
lastPulledHash: entry.lastPulledHash,
165+
lastPulledHash: baseline,
154166
platformHash,
155167
});
156168
const directionTag = `[${direction}]`;
@@ -172,8 +184,8 @@ export async function checkDriftForUpdate(options: {
172184
platformHash,
173185
message:
174186
` ❌ drift detected on ${resourceLabel} ${resourceId} ${directionTag}: ` +
175-
`platform hash (${platformHash.slice(0, 8)}...) differs from last-pulled ` +
176-
`(${entry.lastPulledHash.slice(0, 8)}...). ` +
187+
`platform hash (${platformHash.slice(0, 8)}...) differs from baseline ` +
188+
`(${baseline.slice(0, 8)}...). ` +
177189
`${formatDriftLabel(direction)} ` +
178190
`Re-run pull, resolve locally, or push with --overwrite to take ownership.`,
179191
};

src/hash-store.ts

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Per-developer drift baseline store.
2+
//
3+
// The "last known platform state" content hash for each resource lives here,
4+
// in `.vapi-state-hash/<org>/<uuid>` — one file per resource, filename is the
5+
// platform UUID, contents are the sha256 baseline hash. This is the hash a
6+
// push compares against the live dashboard to decide whether someone changed
7+
// the resource out of band since I last pulled or pushed it.
8+
//
9+
// Design constraints:
10+
// - GITIGNORED, per-developer. The baseline is "what *I* last saw on the
11+
// platform," not a shared fact — it must not be committed.
12+
// - CONFIG-FREE. This module must NOT import `config.ts`, which parses a
13+
// single org and `process.exit(1)`s at module load if no VAPI_TOKEN is
14+
// set. The migration command needs to operate on every org without a
15+
// token, so the org is always passed in explicitly and BASE_DIR is
16+
// computed here the same way `config.ts` computes it.
17+
// - The filename (UUID) is purely organizational. The hash is computed over
18+
// canonical resource CONTENT by callers; this module never inspects it.
19+
20+
import { existsSync, mkdirSync, readFileSync } from "fs";
21+
import { rename, writeFile } from "fs/promises";
22+
import { dirname, join } from "path";
23+
import { fileURLToPath } from "url";
24+
25+
// Same expression as config.ts: the repo root is one level up from src/.
26+
const BASE_DIR = join(dirname(fileURLToPath(import.meta.url)), "..");
27+
28+
// Top-level store directory. Per-org subfolders live beneath it. Exported so
29+
// the migration can locate every org's baselines and so `.gitignore` reasoning
30+
// stays in one place.
31+
export const HASH_STORE_ROOT = join(BASE_DIR, ".vapi-state-hash");
32+
33+
// Guard against a UUID that would escape its org folder. Platform UUIDs never
34+
// contain these, but reading them straight into a path warrants a check.
35+
function assertSafeUuid(uuid: string): void {
36+
if (!uuid || uuid.includes("/") || uuid.includes("\\") || uuid.includes("..")) {
37+
throw new Error(`Refusing to use unsafe hash-store key: ${JSON.stringify(uuid)}`);
38+
}
39+
}
40+
41+
export function hashStoreDir(org: string): string {
42+
return join(HASH_STORE_ROOT, org);
43+
}
44+
45+
function baselinePath(org: string, uuid: string): string {
46+
assertSafeUuid(uuid);
47+
return join(hashStoreDir(org), uuid);
48+
}
49+
50+
// Read the baseline hash for a resource, or undefined if none has been
51+
// recorded yet (fresh clone, never pulled/pushed). Undefined maps to the
52+
// `no-baseline` drift direction — the caller proceeds without blocking.
53+
export function readBaseline(org: string, uuid: string): string | undefined {
54+
const path = baselinePath(org, uuid);
55+
if (!existsSync(path)) return undefined;
56+
const contents = readFileSync(path, "utf-8").trim();
57+
return contents.length > 0 ? contents : undefined;
58+
}
59+
60+
// Write (or overwrite) the baseline hash for a resource. Atomic: emit to a
61+
// sibling temp file then rename over the target, so a crash mid-write can't
62+
// leave a truncated hash that would manufacture phantom drift. Creates the
63+
// org subfolder on demand.
64+
export async function writeBaseline(
65+
org: string,
66+
uuid: string,
67+
hash: string,
68+
): Promise<void> {
69+
const path = baselinePath(org, uuid);
70+
mkdirSync(hashStoreDir(org), { recursive: true });
71+
const tmpPath = `${path}.tmp`;
72+
await writeFile(tmpPath, `${hash}\n`);
73+
await rename(tmpPath, path);
74+
}
75+
76+
// Remove a baseline. Used when a stale state mapping is dropped (the resource
77+
// no longer exists on the platform), mirroring the existing
78+
// `delete stateSection[resourceId]` recovery. Missing file is a no-op.
79+
export async function deleteBaseline(org: string, uuid: string): Promise<void> {
80+
const path = baselinePath(org, uuid);
81+
if (!existsSync(path)) return;
82+
const { rm } = await import("fs/promises");
83+
await rm(path, { force: true });
84+
}

src/migrate-cmd.ts

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// CLI entry: `npm run migrate`
2+
//
3+
// One-time migration of every `.vapi-state.<org>.json` in the repo from the
4+
// legacy fat state file to the slim `name → { uuid }` form + the
5+
// `.vapi-state-hash/<org>/<uuid>` baseline store. Takes NO org argument and
6+
// needs NO VAPI_TOKEN — it's a pure local file operation, so it deliberately
7+
// does not import `config.ts` (which would force an org + token at load).
8+
//
9+
// Idempotent: re-running after migration seeds nothing and rewrites nothing.
10+
11+
import { resolve } from "path";
12+
import { fileURLToPath } from "url";
13+
import { migrateAll } from "./migrate-hash-store.ts";
14+
15+
async function main(): Promise<void> {
16+
console.log(
17+
"═══════════════════════════════════════════════════════════════",
18+
);
19+
console.log("🧬 Vapi GitOps Migrate — state file → hash store");
20+
console.log(
21+
"═══════════════════════════════════════════════════════════════\n",
22+
);
23+
24+
const result = await migrateAll();
25+
26+
if (result.orgs.length === 0) {
27+
console.log(" No .vapi-state.<org>.json files found — nothing to migrate.");
28+
return;
29+
}
30+
31+
let totalSeeded = 0;
32+
let totalSlimmed = 0;
33+
for (const org of result.orgs) {
34+
totalSeeded += org.seeded;
35+
if (org.slimmed) totalSlimmed++;
36+
const status = org.slimmed ? "migrated" : "already slim";
37+
console.log(
38+
` ${org.slimmed ? "✅" : "✔️ "} ${org.stateFile} (${org.org}): ` +
39+
`${status} — seeded ${org.seeded} baseline(s)` +
40+
(org.skipped > 0 ? `, ${org.skipped} already present` : ""),
41+
);
42+
}
43+
44+
console.log(
45+
`\n✅ Migration complete: ${result.orgs.length} org(s), ` +
46+
`${totalSlimmed} state file(s) slimmed, ${totalSeeded} baseline(s) seeded ` +
47+
`into .vapi-state-hash/.`,
48+
);
49+
}
50+
51+
const isMainModule =
52+
process.argv[1] !== undefined &&
53+
resolve(process.argv[1]) === fileURLToPath(import.meta.url);
54+
55+
if (isMainModule) {
56+
main().catch((error) => {
57+
console.error(
58+
"\n❌ Migration failed:",
59+
error instanceof Error ? error.message : error,
60+
);
61+
process.exit(1);
62+
});
63+
}

0 commit comments

Comments
 (0)