feat(appkit): relocate typegen caches to committed .appkit/ dir

Move the query/metric cache (types-cache.json) and serving cache
(serving-types-cache.json) out of node_modules/.databricks/appkit into the
committed per-app .appkit/ directory via getCommittedCacheDir(), so they
survive `pnpm install` and let a blocking CI/CD build succeed with no
warehouse access. Flat filenames (the dir namespaces them). Both writers now
sort top-level record keys for deterministic, merge-friendly diffs.

The serving plugin's runtime read now resolves through the same exported
getServingCachePath() the writer uses (reader == writer by construction,
fixing the inline-path drift risk). Add queryCacheFileExists() for the
blocking-mode bootstrap-vs-drift message (consumed next). Route the ephemeral
ui-variant choices file through getEphemeralStateDir() (path unchanged; stays
gitignored). Document .appkit/ as a committed artifact in the template.

xavier loop: iteration 3 -- Phase 2 Wave 1 (2A, 2B+2C, 2D) + Phase 3A

Co-authored-by: Isaac
Signed-off-by: Atila Fassina <atila@fassina.eu>

Author: atilafassina

packages/appkit/src/plugins/serving/serving.ts

@@ -1,4 +1,3 @@
-import path from "node:path";
 import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
 import type express from "express";
@@ -9,6 +8,7 @@ import { createLogger } from "../../logging";
 import { type ExecutionResult, Plugin, toPlugin } from "../../plugin";
 import type { PluginManifest, ResourceRequirement } from "../../registry";
 import { ResourceType } from "../../registry";
+import { getServingCachePath } from "../../type-generator/serving/cache";
 import { servingInvokeDefaults } from "./defaults";
 import manifest from "./manifest.json";
 import { filterRequestBody, loadEndpointSchemas } from "./schema-filter";
@@ -66,14 +66,7 @@ export class ServingPlugin extends Plugin {
   }
 
   async setup(): Promise<void> {
-    const cacheFile = path.join(
-      process.cwd(),
-      "node_modules",
-      ".databricks",
-      "appkit",
-      ".appkit-serving-types-cache.json",
-    );
-    this.schemaAllowlists = await loadEndpointSchemas(cacheFile);
+    this.schemaAllowlists = await loadEndpointSchemas(getServingCachePath());
     if (this.schemaAllowlists.size > 0) {
       logger.debug(
         "Loaded schema allowlists for %d endpoint(s)",

packages/appkit/src/plugins/ui-variants/choice-sink.ts

@@ -1,22 +1,31 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { getEphemeralStateDir } from "shared";
 
 /**
- * Path of the JSONL choices file, relative to the app's working directory.
- * A coding agent reads it to pick up the developer's in-browser confirmation
- * and finalize the chosen variant.
+ * Filename of the JSONL choices file. The agent skill discovers this file at
+ * the ephemeral state directory (under node_modules/).
+ */
+const UI_CHOICES_FILENAME = ".appkit-ui-choices.jsonl";
+
+/**
+ * Default absolute path to the JSONL choices file: ephemeral state dir
+ * (node_modules/.databricks/appkit/) + the contract filename. Kept
+ * ephemeral & gitignored; NOT part of the committed .appkit/ relocation.
  *
- * Under `node_modules/`, so it's gitignored and cleared on a clean install.
- * Each line is spent on read — the agent removes it once the choice is
- * finalized.
+ * A coding agent reads it to pick up the developer's in-browser confirmation
+ * and finalize the chosen variant. Each line is spent on read — the agent
+ * removes it once the choice is finalized.
  *
  * CONTRACT: the `databricks-app-variants` agent skill (in the
- * databricks-agent-skills repo) discovers the choices file at this path.
- * Changing this value silently breaks that skill's file discovery — update the
- * skill's `find` path in the same change.
+ * databricks-agent-skills repo) discovers the choices file at this absolute
+ * path. Changing this value silently breaks that skill's file discovery —
+ * update the skill's `find` path in the same change.
  */
-const UI_CHOICES_FILE =
-  "node_modules/.databricks/appkit/.appkit-ui-choices.jsonl";
+const DEFAULT_UI_CHOICES_PATH = path.join(
+  getEphemeralStateDir(),
+  UI_CHOICES_FILENAME,
+);
 
 /**
  * One recorded variant choice.
@@ -42,14 +51,16 @@ export interface UiChoiceRecord {
 
 /**
  * File store for confirmed variant choices: upserts choices into
- * {@link UI_CHOICES_FILE}, one line per `<Variants>` id.
+ * {@link DEFAULT_UI_CHOICES_PATH}, one line per `<Variants>` id.
  *
  * The store is **keyed and latest-wins**: at most one record per `blockId`, and
  * recording an existing `blockId` replaces it rather than appending, so the file
  * always reflects the current choice for each block.
  *
- * The file is resolved against `process.cwd()`, so it lands under whatever
- * directory the dev server runs from. Concurrent confirms are serialized behind
+ * By default, the file lands at the ephemeral state directory under
+ * `node_modules/.databricks/appkit/`. Callers may override the path (e.g., for
+ * testing), in which case it is resolved against `process.cwd()`; absolute
+ * paths are passed through unchanged. Concurrent confirms are serialized behind
  * an internal queue so their read-modify-write can't interleave and lose an
  * update.
  *
@@ -59,8 +70,8 @@ export class FileChoiceStore {
   private readonly filePath: string;
   private writeQueue: Promise<void> = Promise.resolve();
 
-  constructor(relativePath: string = UI_CHOICES_FILE) {
-    this.filePath = path.resolve(process.cwd(), relativePath);
+  constructor(pathArg: string = DEFAULT_UI_CHOICES_PATH) {
+    this.filePath = path.resolve(process.cwd(), pathArg);
   }
 
   record(record: UiChoiceRecord): Promise<void> {

packages/appkit/src/type-generator/cache.ts

@@ -1,6 +1,7 @@
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
+import { getCommittedCacheDir } from "shared";
 import { createLogger } from "../logging/logger";
 import type { MetricSchema } from "./mv-registry/types";
 
@@ -38,7 +39,7 @@ export interface MetricCacheEntry {
 /**
  * Structural gate for reviving a cached metric entry at partition time.
  *
- * The cache file lives in `node_modules/.databricks` and is plain JSON —
+ * The cache file lives in the committed `.appkit/` dir and is plain JSON —
  * hand-edits, truncation, or a stale writer can leave entries whose shape no
  * longer matches {@link MetricCacheEntry}. A malformed entry must read as a
  * cache MISS (re-describe) rather than crash the pass or render revived
@@ -87,13 +88,7 @@ interface Cache {
 }
 
 export const CACHE_VERSION = "3";
-const CACHE_FILE = ".appkit-types-cache.json";
-const CACHE_DIR = path.join(
-  process.cwd(),
-  "node_modules",
-  ".databricks",
-  "appkit",
-);
+const CACHE_FILE = "types-cache.json";
 
 /**
  * Hash the SQL query
@@ -120,9 +115,10 @@ export function metricCacheHash(source: string, lane: string): string {
  * @returns - the cache
  */
 export async function loadCache(): Promise<Cache> {
-  const cachePath = path.join(CACHE_DIR, CACHE_FILE);
+  const cacheDir = getCommittedCacheDir();
+  const cachePath = path.join(cacheDir, CACHE_FILE);
   try {
-    await fs.mkdir(CACHE_DIR, { recursive: true });
+    await fs.mkdir(cacheDir, { recursive: true });
 
     const raw = await fs.readFile(cachePath, "utf8");
     const cache = JSON.parse(raw) as Cache;
@@ -138,10 +134,53 @@ export async function loadCache(): Promise<Cache> {
 }
 
 /**
- * Save the cache to the file system
+ * Save the cache to the file system with deterministic serialization
+ * (sorted top-level keys for no-op regen diffs and merge-friendly output).
  * @param cache - cache object to save
  */
 export async function saveCache(cache: Cache): Promise<void> {
-  const cachePath = path.join(CACHE_DIR, CACHE_FILE);
-  await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8");
+  const cacheDir = getCommittedCacheDir();
+  const cachePath = path.join(cacheDir, CACHE_FILE);
+
+  // Ensure the cache directory exists
+  await fs.mkdir(cacheDir, { recursive: true });
+
+  // Rebuild cache with sorted top-level record keys for deterministic output.
+  // This ensures a no-op regen produces an identical file (no spurious diffs)
+  // and unrelated query/metric changes don't collide in git merges.
+  const sorted: Cache = {
+    version: cache.version,
+    queries: Object.fromEntries(
+      Object.keys(cache.queries)
+        .sort()
+        .map((key) => [key, cache.queries[key]]),
+    ),
+  };
+
+  // Include sorted metrics if present
+  if (cache.metrics) {
+    const metrics = cache.metrics;
+    sorted.metrics = Object.fromEntries(
+      Object.keys(metrics)
+        .sort()
+        .map((key) => [key, metrics[key]]),
+    );
+  }
+
+  await fs.writeFile(cachePath, JSON.stringify(sorted, null, 2), "utf8");
+}
+
+/**
+ * Whether the committed query/metric cache file exists on disk. Lets the
+ * blocking-mode fatal path distinguish an uninitialized cache (bootstrap:
+ * run `generate-types --wait` against a warehouse and commit `.appkit/`) from
+ * a drifted key (regenerate the affected query/metric).
+ */
+export async function queryCacheFileExists(): Promise<boolean> {
+  try {
+    await fs.access(path.join(getCommittedCacheDir(), CACHE_FILE));
+    return true;
+  } catch {
+    return false;
+  }
 }

packages/appkit/src/type-generator/serving/cache.ts

@@ -1,18 +1,13 @@
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
+import { getCommittedCacheDir } from "shared";
 import { createLogger } from "../../logging/logger";
 
 const logger = createLogger("type-generator:serving:cache");
 
 export const CACHE_VERSION = "1";
-const CACHE_FILE = ".appkit-serving-types-cache.json";
-const CACHE_DIR = path.join(
-  process.cwd(),
-  "node_modules",
-  ".databricks",
-  "appkit",
-);
+const CACHE_FILE = "serving-types-cache.json";
 
 export interface ServingCacheEntry {
   hash: string;
@@ -27,14 +22,24 @@ export interface ServingCache {
   endpoints: Record<string, ServingCacheEntry>;
 }
 
+/**
+ * Absolute path of the committed serving-types cache. The single source of
+ * truth shared by the writer (this module) and the runtime reader
+ * (serving plugin `setup()`), so a relocation can never desync them.
+ */
+export function getServingCachePath(): string {
+  return path.join(getCommittedCacheDir(), CACHE_FILE);
+}
+
 export function hashSchema(schemaJson: string): string {
   return crypto.createHash("sha256").update(schemaJson).digest("hex");
 }
 
 export async function loadServingCache(): Promise<ServingCache> {
-  const cachePath = path.join(CACHE_DIR, CACHE_FILE);
+  const cachePath = getServingCachePath();
+  const cacheDir = getCommittedCacheDir();
   try {
-    await fs.mkdir(CACHE_DIR, { recursive: true });
+    await fs.mkdir(cacheDir, { recursive: true });
     const raw = await fs.readFile(cachePath, "utf8");
     const cache = JSON.parse(raw) as ServingCache;
     if (cache.version === CACHE_VERSION) {
@@ -50,7 +55,23 @@ export async function loadServingCache(): Promise<ServingCache> {
 }
 
 export async function saveServingCache(cache: ServingCache): Promise<void> {
-  const cachePath = path.join(CACHE_DIR, CACHE_FILE);
-  await fs.mkdir(CACHE_DIR, { recursive: true });
-  await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8");
+  const cachePath = getServingCachePath();
+  const cacheDir = getCommittedCacheDir();
+  await fs.mkdir(cacheDir, { recursive: true });
+
+  // Sort endpoint keys for deterministic output (merge-friendly diffs)
+  const sortedCache: ServingCache = {
+    version: cache.version,
+    endpoints: Object.keys(cache.endpoints)
+      .sort()
+      .reduce(
+        (acc, key) => {
+          acc[key] = cache.endpoints[key];
+          return acc;
+        },
+        {} as Record<string, (typeof cache.endpoints)[string]>,
+      ),
+  };
+
+  await fs.writeFile(cachePath, JSON.stringify(sortedCache, null, 2), "utf8");
 }

packages/appkit/src/type-generator/serving/tests/cache.test.ts

@@ -81,27 +81,43 @@ describe("serving cache", () => {
   });
 
   describe("saveServingCache", () => {
-    test("writes cache to file", async () => {
+    test("writes cache to file with sorted endpoints", async () => {
       vi.mocked(fs.writeFile).mockResolvedValue();
 
       const cache: ServingCache = {
         version: CACHE_VERSION,
         endpoints: {
-          test: {
+          zebra: {
             hash: "xyz",
             requestType: "{}",
             responseType: "{}",
             chunkType: null,
             requestKeys: [],
           },
+          apple: {
+            hash: "abc",
+            requestType: "{}",
+            responseType: "{}",
+            chunkType: null,
+            requestKeys: [],
+          },
         },
       };
 
       await saveServingCache(cache);
 
+      // Endpoints should be sorted alphabetically in the output
+      const expectedSorted: ServingCache = {
+        version: CACHE_VERSION,
+        endpoints: {
+          apple: cache.endpoints.apple,
+          zebra: cache.endpoints.zebra,
+        },
+      };
+
       expect(fs.writeFile).toHaveBeenCalledWith(
-        expect.stringContaining(".appkit-serving-types-cache.json"),
-        JSON.stringify(cache, null, 2),
+        expect.stringContaining("serving-types-cache.json"),
+        JSON.stringify(expectedSorted, null, 2),
         "utf8",
       );
     });