Merge pull request #38 from VapiAI/feat/pull-drift-direction-classifier

feat(drift): three-way drift-direction classifier + content-drift audit + canonicalization fixes

Author: dhruva-vapi

AGENTS.md

@@ -92,6 +92,34 @@ npm run apply -- <org> <path-to-file> # single-file apply (same safety, scoped d
 
 Runs the engine's local validators against every YAML/MD file in the org without any network call. Catches shape errors (missing required fields, wrong types, stale tool references) before they burn a deploy. **Run before every `apply`.**
 
+### `npm run pull -- <org>` — refresh from dashboard (default: plain pull)
+
+**Default to plain pull, NOT --force.** Plain `npm run pull -- <org>` shows:
+- Per-resource direction labels (dashboard-ahead / local-ahead / both-diverged / clean)
+- End-of-pull summary with counts per direction
+- A hard gate (`--resolve` flag) on 3-way conflicts before they silently lose data
+
+`--force` skips all of this and just overwrites local with dashboard. Use it ONLY when you literally need to nuke local and re-materialize dashboard truth (rare). Plain pull is the DEFAULT for both humans and agents; `--force` is the escape hatch.
+
+**Pull-output icon legend.** Distinct semantics in a single pulled-resource line:
+
+| Icon | Meaning |
+|------|---------|
+| `📝` | Engine wrote/updated a file on disk (clean / no-baseline path) |
+| `✨` | Engine created a NEW file on disk (first-time pull of this resource) |
+| `✏️`  | Locally modified file detected by git, preserved as-is |
+| `⬆️`  | `local-ahead` — local has unpushed edits, needs to flow UP to dashboard (preserved) |
+| `⬇️`  | `--resolve=theirs` — overwrote local with dashboard (flowed DOWN) |
+| `🔒` | Platform-default resource (read-only, immutable) |
+| `🚫` | Matched `.vapi-ignore` (not tracked locally) |
+| `🗑️`  | Locally deleted (deletion intent recorded in state) |
+
+Mental model: `⬆️` flows UP (push), `⬇️` flows DOWN (pull-overwrite), `📝` is the engine doing routine file I/O.
+
+**First-adoption noise on customer repos.** The first `npm run audit -- <org>` after upgrading to a version with the content-drift rule will surface `[content-drift] [no-baseline]` info findings for every resource whose state row predates `lastPulledHash` tracking. These are info-severity (do NOT block CI), but the noise is real. Two ways to clear:
+- `npm run pull -- <org> --bootstrap` — refreshes state with the current platform-hash baseline, no resource materialization. One-shot fix for the whole fleet.
+- Or just let them sit — each plain `npm run pull` after this writes `lastPulledHash` for whichever resources it touches, so the no-baseline rows naturally drain over the next few sync cycles.
+
 ### `npm run push -- <org>` — raw push, no pre-pull
 
 Skips the merge pass. Only use when (a) you literally just ran `pull` and (b) you're certain no one has touched the dashboard since. In a multi-developer environment or when dashboard editors are in play, default to `apply` instead. Stale local state can clobber recent dashboard edits or PATCH against UUIDs that no longer exist.

src/audit-cmd.ts

@@ -23,8 +23,14 @@ import { VALID_RESOURCE_TYPES } from "./types.ts";
 
 // Single source of truth for the exit-code contract. Exported so tests can pin
 // behavior without duplicating the predicate.
+//
+// `info`-severity findings (e.g. `no-baseline` from content-drift on fresh
+// clones) are surfaced for operator visibility but do NOT block CI. Without
+// this carve-out, every customer's first audit after this feature lands would
+// fail their build pipeline. `warn` and `error` still exit 1.
 export function exitCodeForFindings(findings: AuditFinding[]): 0 | 1 {
-  return findings.length === 0 ? 0 : 1;
+  const actionable = findings.filter((f) => f.severity !== "info");
+  return actionable.length === 0 ? 0 : 1;
 }
 
 function groupFindings(

src/audit.ts

@@ -10,6 +10,7 @@
 //   5. sibling-base-slug    multiple slugs sharing the same base-slug
 //   6. dashboard-orphan     dashboard UUID not in state (.vapi-ignore suppresses)
 //   7. inline-tools         assistant with non-empty model.tools (use toolIds)
+//   8. content-drift        local vs dashboard vs lastPulledHash direction
 //
 // Designed for dependency injection: state loader, local file lister, remote
 // fetcher, and per-assistant tool reader are all swappable so tests stay
@@ -21,15 +22,18 @@
 import { readFile } from "fs/promises";
 import { join } from "path";
 import { matchesIgnore, RESOURCES_DIR } from "./config.ts";
+import { credentialReverseMap } from "./credentials.ts";
+import { classifyDrift } from "./drift.ts";
 import { findOrphanResourceIds } from "./new-file-gate.ts";
 import {
+  canonicalizeForHash,
   fetchAllResources,
   listExistingResourceIds,
   type VapiResource,
 } from "./pull.ts";
-import { FOLDER_MAP } from "./resources.ts";
+import { FOLDER_MAP, hashLocalResource } from "./resources.ts";
 import { extractBaseSlug, slugify } from "./slug-utils.ts";
-import { loadState } from "./state.ts";
+import { hashPayload, loadState } from "./state.ts";
 import type { ResourceType, StateFile } from "./types.ts";
 import { VALID_RESOURCE_TYPES } from "./types.ts";
 
@@ -45,9 +49,10 @@ export type AuditRule =
   | "sibling-base-slug"
   | "dashboard-orphan"
   | "inline-tools"
+  | "content-drift"
   | "fetch-failed";
 
-export type AuditSeverity = "warn" | "error";
+export type AuditSeverity = "info" | "warn" | "error";
 
 export interface AuditFinding {
   severity: AuditSeverity;
@@ -345,6 +350,89 @@ function checkDashboardOrphans(
   return findings;
 }
 
+function contentDriftSuggestedAction(direction: string): string {
+  if (direction === "dashboard-ahead") {
+    return "run npm run pull to refresh, or push --overwrite to take ownership";
+  }
+  if (direction === "local-ahead") {
+    return "run npm run push to sync up";
+  }
+  if (direction === "both-diverged") {
+    return "run npm run pull --resolve=ours|theirs|fail to resolve";
+  }
+  if (direction === "no-baseline") {
+    return "run npm run pull --bootstrap to seed lastPulledHash baseline";
+  }
+  return "no action needed";
+}
+
+function checkContentDrift(
+  type: ResourceType,
+  state: StateFile,
+  remote: VapiResource[],
+  localIds: string[],
+): AuditFinding[] {
+  const remoteByUuid = new Map(remote.map((r) => [r.id, r]));
+  const credReverse = credentialReverseMap(state);
+  const findings: AuditFinding[] = [];
+
+  for (const resourceId of localIds) {
+    const entry = state[type][resourceId];
+    if (!entry) continue;
+
+    const localHash = hashLocalResource(type, resourceId);
+    if (!localHash) continue;
+
+    const remoteResource = remoteByUuid.get(entry.uuid);
+    if (!remoteResource) continue;
+
+    // canonicalizeForHash centralizes the 3-step pipeline + _platformDefault
+    // mutation so the hash agrees with pull-write's lastPulledHash basis.
+    const platformHash = hashPayload(
+      canonicalizeForHash(remoteResource, state, credReverse),
+    );
+    const direction = classifyDrift({
+      localHash,
+      lastPulledHash: entry.lastPulledHash,
+      platformHash,
+    });
+    if (direction === "clean") continue;
+
+    // Severity mapping:
+    //   - both-diverged  → error (CI-blocking; needs --resolve to disambiguate)
+    //   - local-ahead    → warn  (un-pushed edits; operator should push)
+    //   - dashboard-ahead → warn (UI edits not pulled; operator should pull)
+    //   - no-baseline    → info  (expected on fresh clones / pre-bootstrap;
+    //                             surfaces what would otherwise be silently
+    //                             skipped, but does NOT block CI)
+    const severity =
+      direction === "both-diverged"
+        ? "error"
+        : direction === "no-baseline"
+          ? "info"
+          : "warn";
+    const detail =
+      direction === "local-ahead"
+        ? "local has unpushed edits"
+        : direction === "dashboard-ahead"
+          ? "local diverges from dashboard since last pull"
+          : direction === "both-diverged"
+            ? "3-way conflict between local, dashboard, and last-pulled baseline"
+            : "no lastPulledHash baseline — content-drift coverage uninitialized";
+
+    findings.push({
+      severity,
+      type,
+      rule: "content-drift",
+      resourceIds: [resourceId],
+      message: `${detail} [${direction}]`,
+      suggestedAction: contentDriftSuggestedAction(direction),
+    });
+  }
+
+  return findings;
+}
+
 async function checkInlineTools(
   state: StateFile,
   readAssistantTools: (resourceId: string) => unknown[] | Promise<unknown[]>,
@@ -434,6 +522,7 @@ export async function runAudit(
       const remoteUuids = new Set(remote.map((r) => r.id));
       findings.push(...checkStateGhosts(type, state, remoteUuids));
       findings.push(...checkDashboardOrphans(type, state, remote));
+      findings.push(...checkContentDrift(type, state, remote, localIds));
     }
 
     findings.push(...checkStateUuidCollisions(type, state));
@@ -457,7 +546,8 @@ export async function runAudit(
 // ─────────────────────────────────────────────────────────────────────────────
 
 export function formatFinding(f: AuditFinding): string {
-  const icon = f.severity === "error" ? "❌" : "⚠️ ";
+  const icon =
+    f.severity === "error" ? "❌" : f.severity === "info" ? "ℹ️ " : "⚠️ ";
   const uuidSuffix = f.uuid ? ` [uuid=${f.uuid}]` : "";
   const lines = [
     `  ${icon} [${f.rule}] ${f.type}/${f.resourceIds.join(", ")}${uuidSuffix}`,
@@ -473,5 +563,8 @@ export function summarizeFindings(findings: AuditFinding[]): string {
   if (findings.length === 0) return "✅ No audit findings.";
   const errors = findings.filter((f) => f.severity === "error").length;
   const warns = findings.filter((f) => f.severity === "warn").length;
-  return `📋 Audit: ${findings.length} finding(s) — ${errors} error(s), ${warns} warning(s)`;
+  const infos = findings.filter((f) => f.severity === "info").length;
+  const parts = [`${errors} error(s)`, `${warns} warning(s)`];
+  if (infos > 0) parts.push(`${infos} info finding(s)`);
+  return `📋 Audit: ${findings.length} finding(s) — ${parts.join(", ")}`;
 }

src/drift.ts

@@ -22,6 +22,45 @@ import { VAPI_BASE_URL, VAPI_TOKEN } from "./config.ts";
 import { hashPayload } from "./state-serialize.ts";
 import type { ResourceState } from "./types.ts";
 
+export type DriftDirection =
+  | "clean"
+  | "dashboard-ahead"
+  | "local-ahead"
+  | "both-diverged"
+  | "no-baseline";
+
+export interface ClassifyDriftInput {
+  localHash: string;
+  lastPulledHash?: string;
+  platformHash: string;
+}
+
+export function classifyDrift(input: ClassifyDriftInput): DriftDirection {
+  const { localHash, lastPulledHash, platformHash } = input;
+  if (!lastPulledHash) return "no-baseline";
+  const localMatches = localHash === lastPulledHash;
+  const platformMatches = platformHash === lastPulledHash;
+  if (localMatches && platformMatches) return "clean";
+  if (localMatches && !platformMatches) return "dashboard-ahead";
+  if (!localMatches && platformMatches) return "local-ahead";
+  return "both-diverged";
+}
+
+export function formatDriftLabel(direction: DriftDirection): string {
+  switch (direction) {
+    case "dashboard-ahead":
+      return "[dashboard-ahead — sync down via plain pull (preserves local) or push --overwrite to take ownership]";
+    case "local-ahead":
+      return "[local-ahead — run npm run push to propagate local edits up]";
+    case "both-diverged":
+      return "[both-diverged — 3-way conflict, pass --resolve=ours|theirs|fail]";
+    case "no-baseline":
+      return "[direction unknown — no lastPulledHash baseline; pull --bootstrap first]";
+    case "clean":
+      return "[no drift]";
+  }
+}
+
 export interface DriftCheckResult {
   ok: boolean;
   reason: "no-baseline" | "match" | "drift-overwritten" | "drift-blocked";
@@ -73,14 +112,20 @@ function stripServerFields(payload: unknown): unknown {
   return out;
 }
 
+export function hashPlatformResource(resource: unknown): string {
+  return hashPayload(stripServerFields(resource));
+}
+
 export async function checkDriftForUpdate(options: {
   endpoint: string; // e.g. "/assistant/<uuid>"
   resourceLabel: string; // for log lines
   resourceId: string; // local resource id
   state: ResourceState;
   overwrite: boolean;
+  localHash?: string;
 }): Promise<DriftCheckResult> {
-  const { endpoint, resourceLabel, resourceId, state, overwrite } = options;
+  const { endpoint, resourceLabel, resourceId, state, overwrite, localHash } =
+    options;
 
   if (!state.lastPulledHash) {
     return {
@@ -104,14 +149,22 @@ export async function checkDriftForUpdate(options: {
     return { ok: true, reason: "match", platformHash };
   }
 
+  const effectiveLocalHash = localHash ?? state.lastPulledHash ?? platformHash;
+  const direction = classifyDrift({
+    localHash: effectiveLocalHash,
+    lastPulledHash: state.lastPulledHash,
+    platformHash,
+  });
+  const directionTag = `[${direction}]`;
+
   if (overwrite) {
     return {
       ok: true,
       reason: "drift-overwritten",
       platformHash,
       message:
-        `   ⚠️  drift on ${resourceLabel} ${resourceId}: platform changed since last pull, ` +
-        `overwriting (--overwrite).`,
+        `   ⚠️  drift on ${resourceLabel} ${resourceId} ${directionTag}: platform changed since last pull, ` +
+        `overwriting (--overwrite). ${formatDriftLabel(direction)}`,
     };
   }
 
@@ -120,9 +173,10 @@ export async function checkDriftForUpdate(options: {
     reason: "drift-blocked",
     platformHash,
     message:
-      `   ❌ drift detected on ${resourceLabel} ${resourceId}: ` +
+      `   ❌ drift detected on ${resourceLabel} ${resourceId} ${directionTag}: ` +
       `platform hash (${platformHash.slice(0, 8)}...) differs from last-pulled ` +
       `(${state.lastPulledHash.slice(0, 8)}...). ` +
+      `${formatDriftLabel(direction)} ` +
       `Re-run pull, resolve locally, or push with --overwrite to take ownership.`,
   };
 }

src/pull-cmd.ts

@@ -1,6 +1,9 @@
 // Entry point for `npm run pull`. Detects whether an org slug was provided:
 // - With slug: forwards to pull.ts (existing non-interactive behavior)
 // - Without slug: enters interactive mode (org selection + resource picker)
+//
+// Pull flags (--force, --bootstrap, --resolve=ours|theirs|fail) are parsed
+// inside runPull from process.argv, same as --force.
 
 const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
 const arg = process.argv[2];