refactor: smart sync adapters — hash-based diff instead of full copy (#966)

jackwener · web-flow · commit 3c088da53ed0 · 2026-04-11T21:28:40.000+08:00
* refactor: smart sync adapters instead of full copy (#sparse-override) Replace unconditional full-copy of all adapters to ~/.opencli/clis/ with hash-based smart sync that only copies files whose content has changed. Changes: - fetch-adapters.js: use SHA-256 content hashes to skip unchanged files; store per-file hashes in adapter-manifest.json - discovery.ts: simplify ensureUserAdapters() to only create the directory (no longer triggers full copy on first run) - main.ts: fix fast completion to check manifest file existence instead of directory existence (sparse override may have empty user dir) - cli.ts: add `opencli adapter eject/reset/status` commands for managing local adapter overrides - engine.test.ts: add tests for empty user dir and ensureUserAdapters * fix: address review blockers — site-level sync + reset --all 1. Fix `adapter reset --all`: change <site> from required to optional argument so --all can be used without specifying a site name. 2. Change smart sync from file-level to site-level granularity: if any file in a site has changed upstream, overwrite the entire site directory. This matches the agreed product semantics — local modifications to any file in a site are replaced when upstream updates that site. * fix: delete old site dir before writing updated adapter files When a site has upstream changes, delete the entire site directory first, then write the new version. This prevents stale files from older versions lingering in the user directory. * fix: reset --all preserves custom sites, only removes official overrides Blocker 3 fix: reset --all now checks BUILTIN_CLIS to identify official sites and only deletes those, preserving user-created custom sites. * refactor: sparse sync deletes local overrides instead of copying new versions Changed fetch-adapters.js semantics per team agreement: - When an official site has upstream changes, DELETE the local override instead of copying the new version into ~/.opencli/clis/ - Runtime automatically falls back to package baseline - ~/.opencli/clis/ becomes a true sparse override layer * fix: reset <site> rejects custom sites, only allows official overrides Single-site reset now checks BUILTIN_CLIS before deleting, matching the same protection that reset --all already has. * fix: reset <site> allows custom sites per product decision Per @WAWQAQ: explicit single-site reset should work on custom sites too. Differentiate messaging: official sites say "using official baseline", custom sites say "removed custom site". reset --all still only removes official overrides (bulk safety). * fix: reset --all deletes all local sites including custom per product decision Per @WAWQAQ: --all should clear the entire local working cache, including custom sites. Single-site reset already handles both types.
diff --git a/scripts/fetch-adapters.js b/scripts/fetch-adapters.js
@@ -1,21 +1,26 @@
 #!/usr/bin/env node
 
 /**
- * Copy official CLI adapters from the installed package to ~/.opencli/clis/.
+ * Sparse adapter sync: keeps ~/.opencli/clis/ clean by removing stale overrides.
  *
- * Update strategy (file-level granularity via adapter-manifest.json):
- * - Official files (in new manifest) are unconditionally overwritten
- * - Removed official files (in old manifest but not new) are cleaned up
- * - User-created files (never in any manifest) are preserved
- * - Skips if already installed at the same version
+ * Strategy (hash-based, site-level granularity):
+ * - When an official site has upstream changes: DELETE the local override
+ *   (do NOT copy new version — runtime falls back to package baseline)
+ * - When an official site has no changes: leave local override intact
+ * - User-created custom sites (not in package): always preserved
+ * - Skips entirely if already synced at the same version
+ *
+ * ~/.opencli/clis/ is a sparse override layer, not a full copy.
+ * Only eject-ed or user-modified sites appear here.
  *
  * Only runs on global install (npm install -g) or explicit OPENCLI_FETCH=1.
- * No network calls — copies directly from clis/ in the installed package.
+ * No network calls — reads hashes from clis/ in the installed package.
  *
  * This is an ESM script (package.json type: module). No TypeScript, no src/ imports.
  */
 
-import { existsSync, mkdirSync, rmSync, cpSync, readFileSync, writeFileSync, readdirSync, statSync, unlinkSync } from 'node:fs';
+import { existsSync, mkdirSync, rmSync, readFileSync, writeFileSync, readdirSync, statSync, unlinkSync } from 'node:fs';
+import { createHash } from 'node:crypto';
 import { join, resolve, dirname } from 'node:path';
 import { homedir } from 'node:os';
 
@@ -38,7 +43,14 @@ function getPackageVersion() {
 }
 
 /**
- * Read existing manifest. Returns { version, files } or null.
+ * Compute SHA-256 hash of file content.
+ */
+function fileHash(filePath) {
+  return createHash('sha256').update(readFileSync(filePath)).digest('hex');
+}
+
+/**
+ * Read existing manifest. Returns { version, files, hashes } or null.
  */
 function readManifest() {
   try {
@@ -101,30 +113,48 @@ export function fetchAdapters() {
 
   const newOfficialFiles = new Set(walkFiles(BUILTIN_CLIS));
   const oldOfficialFiles = new Set(oldManifest?.files ?? []);
+  const oldHashes = oldManifest?.hashes ?? {};
   mkdirSync(USER_CLIS_DIR, { recursive: true });
 
-  // 1. Copy official files (unconditionally overwrite)
-  let copied = 0;
+  // 1. Compute new hashes and detect which sites have changes
+  const newHashes = {};
+  const siteFiles = new Map(); // site -> [relPath, ...]
   for (const relPath of newOfficialFiles) {
     const src = join(BUILTIN_CLIS, relPath);
-    const dst = join(USER_CLIS_DIR, relPath);
-    mkdirSync(dirname(dst), { recursive: true });
-    cpSync(src, dst, { force: true });
-    copied++;
+    const srcHash = fileHash(src);
+    newHashes[relPath] = srcHash;
+
+    const site = relPath.split('/')[0];
+    if (!siteFiles.has(site)) siteFiles.set(site, []);
+    siteFiles.get(site).push(relPath);
   }
 
-  // 2. Remove files that were official but are no longer (upstream deleted)
-  let removed = 0;
+  // Determine which sites have any changed/new/removed files
+  const changedSites = new Set();
+  for (const [site, files] of siteFiles) {
+    for (const relPath of files) {
+      if (oldHashes[relPath] !== newHashes[relPath]) {
+        changedSites.add(site);
+        break;
+      }
+    }
+  }
+  // Also mark sites that had files removed
   for (const relPath of oldOfficialFiles) {
     if (!newOfficialFiles.has(relPath)) {
-      const dst = join(USER_CLIS_DIR, relPath);
-      try {
-        unlinkSync(dst);
-        pruneEmptyDirs(dst, USER_CLIS_DIR);
-        removed++;
-      } catch {
-        // File may not exist locally
-      }
+      changedSites.add(relPath.split('/')[0]);
+    }
+  }
+
+  // 2. Sparse cleanup: for changed/removed official sites, delete local overrides.
+  //    Do NOT copy new versions — runtime falls back to package baseline.
+  //    Only eject-ed sites live in ~/.opencli/clis/.
+  let cleared = 0;
+  for (const site of changedSites) {
+    const siteDir = join(USER_CLIS_DIR, site);
+    if (existsSync(siteDir)) {
+      rmSync(siteDir, { recursive: true, force: true });
+      cleared++;
     }
   }
 
@@ -206,15 +236,16 @@ export function fetchAdapters() {
     log(`Cleaned up${legacyCleaned > 0 ? ` ${legacyCleaned} legacy shim files` : ''}${tmpCleaned > 0 ? `${legacyCleaned > 0 ? ',' : ''} ${tmpCleaned} stale tmp files` : ''}`);
   }
 
-  // 6. Write updated manifest
+  // 6. Write updated manifest (with per-file hashes for smart sync)
   writeFileSync(MANIFEST_PATH, JSON.stringify({
     version: currentVersion,
     files: [...newOfficialFiles].sort(),
+    hashes: newHashes,
     updatedAt: new Date().toISOString(),
   }, null, 2));
 
-  log(`Installed ${copied} adapter files to ${USER_CLIS_DIR}` +
-    (removed > 0 ? `, removed ${removed} deprecated files` : ''));
+  log(`Synced adapters: ${cleared} local override(s) cleared` +
+    (tsCleaned > 0 ? `, ${tsCleaned} stale .ts files removed` : ''));
 }
 
 function main() {
diff --git a/src/cli.ts b/src/cli.ts
@@ -935,6 +935,121 @@ cli({
       }
     });
 
+  // ── Built-in: adapter management ─────────────────────────────────────────
+  const adapterCmd = program.command('adapter').description('Manage CLI adapters');
+
+  adapterCmd
+    .command('status')
+    .description('Show which sites have local overrides vs using official baseline')
+    .action(async () => {
+      const os = await import('node:os');
+      const userClisDir = path.join(os.homedir(), '.opencli', 'clis');
+      const builtinClisDir = BUILTIN_CLIS;
+      try {
+        const userEntries = await fs.promises.readdir(userClisDir, { withFileTypes: true });
+        const userSites = userEntries.filter(e => e.isDirectory()).map(e => e.name).sort();
+        let builtinSites: string[] = [];
+        try {
+          const builtinEntries = await fs.promises.readdir(builtinClisDir, { withFileTypes: true });
+          builtinSites = builtinEntries.filter(e => e.isDirectory()).map(e => e.name).sort();
+        } catch { /* no builtin dir */ }
+
+        if (userSites.length === 0) {
+          console.log('No local adapter overrides. All sites use the official baseline.');
+          return;
+        }
+
+        console.log(`Local overrides in ~/.opencli/clis/ (${userSites.length} sites):\n`);
+        for (const site of userSites) {
+          const isOfficial = builtinSites.includes(site);
+          const label = isOfficial ? 'override' : 'custom';
+          console.log(`  ${site} [${label}]`);
+        }
+        console.log(`\nOfficial baseline: ${builtinSites.length} sites in package`);
+      } catch {
+        console.log('No local adapter overrides. All sites use the official baseline.');
+      }
+    });
+
+  adapterCmd
+    .command('eject')
+    .description('Copy an official adapter to ~/.opencli/clis/ for local editing')
+    .argument('<site>', 'Site name (e.g. twitter, bilibili)')
+    .action(async (site: string) => {
+      const os = await import('node:os');
+      const userClisDir = path.join(os.homedir(), '.opencli', 'clis');
+      const builtinSiteDir = path.join(BUILTIN_CLIS, site);
+      const userSiteDir = path.join(userClisDir, site);
+
+      try {
+        await fs.promises.access(builtinSiteDir);
+      } catch {
+        console.error(styleText('red', `Error: Site "${site}" not found in official adapters.`));
+        process.exitCode = EXIT_CODES.USAGE_ERROR;
+        return;
+      }
+
+      try {
+        await fs.promises.access(userSiteDir);
+        console.error(styleText('yellow', `Site "${site}" already exists in ~/.opencli/clis/. Use "opencli adapter reset ${site}" first to restore official version.`));
+        process.exitCode = EXIT_CODES.USAGE_ERROR;
+        return;
+      } catch { /* good, doesn't exist yet */ }
+
+      fs.cpSync(builtinSiteDir, userSiteDir, { recursive: true });
+      console.log(styleText('green', `✅ Ejected "${site}" to ~/.opencli/clis/${site}/`));
+      console.log('You can now edit the adapter files. Changes take effect immediately.');
+      console.log(styleText('yellow', 'Note: Official updates to this adapter will overwrite your changes.'));
+    });
+
+  adapterCmd
+    .command('reset')
+    .description('Remove local override and restore official adapter version')
+    .argument('[site]', 'Site name (e.g. twitter, bilibili)')
+    .option('--all', 'Reset all local overrides')
+    .action(async (site: string | undefined, opts: { all?: boolean }) => {
+      const os = await import('node:os');
+      const userClisDir = path.join(os.homedir(), '.opencli', 'clis');
+
+      if (opts.all) {
+        try {
+          const userEntries = await fs.promises.readdir(userClisDir, { withFileTypes: true });
+          const dirs = userEntries.filter(e => e.isDirectory());
+          if (dirs.length === 0) {
+            console.log('No local sites to reset.');
+            return;
+          }
+          for (const dir of dirs) {
+            fs.rmSync(path.join(userClisDir, dir.name), { recursive: true, force: true });
+          }
+          console.log(styleText('green', `✅ Reset ${dirs.length} site(s). All adapters now use official baseline.`));
+        } catch {
+          console.log('No local sites to reset.');
+        }
+        return;
+      }
+
+      if (!site) {
+        console.error(styleText('red', 'Error: Please specify a site name or use --all.'));
+        process.exitCode = EXIT_CODES.USAGE_ERROR;
+        return;
+      }
+
+      const userSiteDir = path.join(userClisDir, site);
+      try {
+        await fs.promises.access(userSiteDir);
+      } catch {
+        console.error(styleText('yellow', `Site "${site}" has no local override.`));
+        return;
+      }
+
+      const isOfficial = fs.existsSync(path.join(BUILTIN_CLIS, site));
+      fs.rmSync(userSiteDir, { recursive: true, force: true });
+      console.log(styleText('green', isOfficial
+        ? `✅ Reset "${site}". Now using official baseline.`
+        : `✅ Removed custom site "${site}".`));
+    });
+
   // ── Built-in: daemon ──────────────────────────────────────────────────────
   const daemonCmd = program.command('daemon').description('Manage the opencli daemon');
   daemonCmd
diff --git a/src/discovery.ts b/src/discovery.ts
@@ -16,7 +16,7 @@ import { type InternalCliCommand, Strategy, registerCommand } from './registry.j
 import { getErrorMessage } from './errors.js';
 import { log } from './logger.js';
 import type { ManifestEntry } from './build-manifest.js';
-import { findPackageRoot, getCliManifestPath, getFetchAdaptersScriptPath } from './package-paths.js';
+import { findPackageRoot, getCliManifestPath } from './package-paths.js';
 
 /** User runtime directory: ~/.opencli */
 export const USER_OPENCLI_DIR = path.join(os.homedir(), '.opencli');
@@ -77,42 +77,15 @@ export async function ensureUserCliCompatShims(baseDir: string = USER_OPENCLI_DI
   }
 }
 
-const ADAPTER_MANIFEST_PATH = path.join(USER_OPENCLI_DIR, 'adapter-manifest.json');
-
 /**
- * First-run fallback: if postinstall was skipped (--ignore-scripts) or failed,
- * trigger adapter fetch on first CLI invocation when ~/.opencli/clis/ is empty.
+ * Ensure the user adapters directory exists.
+ *
+ * With smart sync, ~/.opencli/clis/ only holds files that differ from the
+ * package baseline (upstream-synced cache + autofix output + user overrides).
+ * Built-in adapters are loaded directly from the installed package.
  */
 export async function ensureUserAdapters(): Promise<void> {
-  // If adapter manifest already exists, adapters were fetched — nothing to do
-  try {
-    await fs.promises.access(ADAPTER_MANIFEST_PATH);
-    return;
-  } catch {
-    // No manifest — first run or postinstall was skipped
-  }
-
-  // Check if clis dir has any content (could be manually populated)
-  try {
-    const entries = await fs.promises.readdir(USER_CLIS_DIR);
-    if (entries.length > 0) return;
-  } catch {
-    // Dir doesn't exist — needs fetch
-  }
-
-  log.info('First run detected — copying adapters (one-time setup)...');
-  try {
-    const { execFileSync } = await import('node:child_process');
-    const scriptPath = getFetchAdaptersScriptPath(PACKAGE_ROOT);
-    execFileSync(process.execPath, [scriptPath], {
-      stdio: 'inherit',
-      env: { ...process.env, _OPENCLI_FIRST_RUN: '1' },
-      timeout: 120_000,
-    });
-  } catch (err) {
-    log.warn(`Could not fetch adapters on first run: ${getErrorMessage(err)}`);
-    log.warn('Built-in adapters from the package will be used.');
-  }
+  await fs.promises.mkdir(USER_CLIS_DIR, { recursive: true });
 }
 
 /**
diff --git a/src/engine.test.ts b/src/engine.test.ts
@@ -1,5 +1,5 @@
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { discoverClis, discoverPlugins, ensureUserCliCompatShims, PLUGINS_DIR } from './discovery.js';
+import { discoverClis, discoverPlugins, ensureUserCliCompatShims, ensureUserAdapters, PLUGINS_DIR } from './discovery.js';
 import { executeCommand } from './execution.js';
 import { getRegistry, cli, Strategy } from './registry.js';
 import { clearAllHooks, onAfterExecute } from './hooks.js';
@@ -114,6 +114,34 @@ cli({
   });
 });
 
+describe('ensureUserAdapters', () => {
+  it('creates user clis directory without triggering full copy', async () => {
+    const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-ensure-'));
+    const clisDir = path.join(tempDir, 'clis');
+    try {
+      // Patch USER_CLIS_DIR is not easy, so we test the function behavior indirectly:
+      // ensureUserAdapters should not throw and should be very fast (no fetch script)
+      const start = Date.now();
+      await ensureUserAdapters();
+      const elapsed = Date.now() - start;
+      // Should complete quickly (< 1s) since it only creates a directory
+      expect(elapsed).toBeLessThan(1000);
+    } finally {
+      await fs.promises.rm(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  it('discoverClis handles empty user directory gracefully', async () => {
+    const emptyDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-empty-'));
+    try {
+      // Should not throw for an empty directory (no adapters to discover)
+      await expect(discoverClis(emptyDir)).resolves.not.toThrow();
+    } finally {
+      await fs.promises.rm(emptyDir, { recursive: true, force: true });
+    }
+  });
+});
+
 describe('discoverPlugins', () => {
   const testPluginDir = path.join(PLUGINS_DIR, '__test-plugin__');
   const yamlPath = path.join(testPluginDir, 'greeting.yaml');
diff --git a/src/main.ts b/src/main.ts
@@ -51,10 +51,11 @@ if (argv[0] === 'completion' && argv.length >= 2) {
 // Fast path: --get-completions — read from manifest, skip discovery
 const getCompIdx = process.argv.indexOf('--get-completions');
 if (getCompIdx !== -1) {
-  // Only require manifest for directories that actually exist.
-  // If user clis dir doesn't exist, there are no user adapters to miss.
+  // Only include manifests that actually exist on disk.
+  // With sparse override, the user clis dir may exist but have no manifest.
   const manifestPaths = [getCliManifestPath(BUILTIN_CLIS)];
-  try { fs.accessSync(USER_CLIS); manifestPaths.push(getCliManifestPath(USER_CLIS)); } catch { /* no user dir */ }
+  const userManifest = getCliManifestPath(USER_CLIS);
+  try { fs.accessSync(userManifest); manifestPaths.push(userManifest); } catch { /* no user manifest */ }
   if (hasAllManifests(manifestPaths)) {
     const rest = process.argv.slice(getCompIdx + 1);
     let cursor: number | undefined;
