cache package manifests during intent discovery

Author: LadyBluenotes

packages/intent/src/commands/list.ts

@@ -27,6 +27,8 @@ function printListDebug(result: IntentSkillList): void {
     ['skills', result.debug.skillCount],
     ['warnings', result.debug.warningCount],
     ['conflicts', result.debug.conflictCount],
+    ['packageJsonReadCount', result.debug.scan.packageJsonReadCount],
+    ['packageJsonCacheHits', result.debug.scan.packageJsonCacheHits],
   ])
 }
 

packages/intent/src/commands/load.ts

@@ -28,6 +28,8 @@ function printLoadDebug(loaded: LoadedIntentSkill | ResolvedIntentSkill): void {
     ['skill', loaded.debug.skillName],
     ['path', loaded.debug.path],
     ['warnings', loaded.debug.warningCount],
+    ['packageJsonReadCount', loaded.debug.scan.packageJsonReadCount],
+    ['packageJsonCacheHits', loaded.debug.scan.packageJsonCacheHits],
   ])
 }
 

packages/intent/src/core.ts

@@ -6,6 +6,7 @@ import {
   isPackageExcluded,
   warningMentionsPackage,
 } from './core/excludes.js'
+import { createIntentFsCache, type IntentFsCache } from './fs-cache.js'
 import { rewriteLoadedSkillMarkdownDestinations } from './core/markdown.js'
 import { resolveSkillUseFastPath } from './core/load-resolution.js'
 import { resolveProjectContext } from './core/project-context.js'
@@ -80,6 +81,13 @@ function getScanScope(options: ScanOptions): ScanScope {
   return options.scope ?? (options.includeGlobal ? 'local-and-global' : 'local')
 }
 
+function withFsCache(
+  options: ScanOptions,
+  fsCache: IntentFsCache,
+): ScanOptions & { fsCache: IntentFsCache } {
+  return { ...options, fsCache }
+}
+
 function resolveCoreCwd(options: IntentCoreOptions): string {
   return resolve(process.cwd(), options.cwd ?? process.cwd())
 }
@@ -89,8 +97,9 @@ export function listIntentSkills(
 ): IntentSkillList {
   const cwd = resolveCoreCwd(options)
   const scanOptions = toScanOptions(options)
+  const fsCache = createIntentFsCache()
   const projectContext = resolveProjectContext({ cwd })
-  const scanResult = scanForIntents(cwd, scanOptions)
+  const scanResult = scanForIntents(cwd, withFsCache(scanOptions, fsCache))
   const excludePatterns = getEffectiveExcludePatterns(options, projectContext)
   const excludeMatchers = compileExcludePatterns(excludePatterns)
   const excludedPackages = scanResult.packages
@@ -144,6 +153,7 @@ export function listIntentSkills(
       skillCount: result.skills.length,
       warningCount: result.warnings.length,
       conflictCount: result.conflicts.length,
+      scan: scanResult.stats ?? fsCache.getStats(),
     }
   }
 
@@ -220,12 +230,14 @@ function toResolvedIntentSkill(
 function createLoadedSkillDebug({
   cwd,
   excludes,
+  scan,
   resolution,
   resolved,
   scope,
 }: {
   cwd: string
   excludes: Array<string>
+  scan: LoadedIntentSkillDebug['scan']
   resolution: LoadedIntentSkillDebug['resolution']
   resolved: ResolveSkillResult
   scope: ScanScope
@@ -241,6 +253,7 @@ function createLoadedSkillDebug({
     source: resolved.source,
     path: resolved.path,
     warningCount: resolved.warnings.length,
+    scan,
   }
 }
 
@@ -263,6 +276,7 @@ function resolveIntentSkillInCwd(
     )
   }
 
+  const fsCache = createIntentFsCache()
   const projectContext = resolveProjectContext({ cwd })
   const excludePatterns = getEffectiveExcludePatterns(options, projectContext)
   const excludeMatchers = compileExcludePatterns(excludePatterns)
@@ -281,6 +295,7 @@ function resolveIntentSkillInCwd(
     options,
     projectContext,
     cwd,
+    fsCache,
   )
   if (fastPathResolved) {
     return toResolvedIntentSkill(
@@ -293,13 +308,14 @@ function resolveIntentSkillInCwd(
             excludes: excludePatterns,
             resolution: 'fast-path',
             resolved: fastPathResolved,
+            scan: fsCache.getStats(),
             scope,
           })
         : undefined,
     )
   }
 
-  const scanResult = scanForIntents(cwd, scanOptions)
+  const scanResult = scanForIntents(cwd, withFsCache(scanOptions, fsCache))
   let resolved: ReturnType<typeof resolveSkillUse>
   try {
     resolved = resolveSkillUse(use, scanResult)
@@ -322,6 +338,7 @@ function resolveIntentSkillInCwd(
           excludes: excludePatterns,
           resolution: 'full-scan',
           resolved,
+          scan: scanResult.stats ?? fsCache.getStats(),
           scope,
         })
       : undefined,

packages/intent/src/core/load-resolution.ts

@@ -1,11 +1,11 @@
 import { existsSync } from 'node:fs'
 import { dirname, join, resolve } from 'node:path'
+import { createIntentFsCache, type IntentFsCache } from '../fs-cache.js'
 import { resolveSkillEntry, type ResolveSkillResult } from '../resolver.js'
 import { scanIntentPackageAtRoot } from '../scanner.js'
-import { resolveWorkspacePackages } from '../workspace-patterns.js'
+import { findWorkspacePackages } from '../workspace-patterns.js'
 import { getDeps, resolveDepDir } from '../utils.js'
 import { warningMentionsPackage } from './excludes.js'
-import { readPackageJson } from './package-json.js'
 import {
   resolveProjectContext,
   type ProjectContext,
@@ -21,6 +21,7 @@ interface WorkspacePackageInfo {
 
 function readWorkspacePackageInfos(
   context: ProjectContext,
+  fsCache: IntentFsCache,
 ): Array<WorkspacePackageInfo> {
   const dirs = new Set<string>()
 
@@ -31,16 +32,13 @@ function readWorkspacePackageInfos(
   if (context.workspaceRoot) {
     dirs.add(context.workspaceRoot)
 
-    for (const dir of resolveWorkspacePackages(
-      context.workspaceRoot,
-      context.workspacePatterns,
-    )) {
+    for (const dir of findWorkspacePackages(context.workspaceRoot)) {
       dirs.add(dir)
     }
   }
 
   return [...dirs].flatMap((dir) => {
-    const packageJson = readPackageJson(dir)
+    const packageJson = fsCache.readPackageJson(dir)
     if (!packageJson) return []
 
     return [
@@ -154,10 +152,11 @@ function getDirectLoadFastPathCandidateDirs(
 function getWorkspaceLoadFastPathCandidateDirs(
   packageName: string,
   context: ProjectContext,
+  fsCache: IntentFsCache,
 ): Array<string> {
   const candidates: Array<string> = []
   const seen = new Set<string>()
-  const workspacePackages = readWorkspacePackageInfos(context)
+  const workspacePackages = readWorkspacePackageInfos(context, fsCache)
 
   for (const pkg of workspacePackages) {
     if (pkg.name === packageName) {
@@ -212,10 +211,12 @@ function resolveFromPackageRoots(
   packageRoots: Array<string>,
   parsedUse: SkillUse,
   cwd: string,
+  fsCache: IntentFsCache,
 ): ResolveSkillResult | null {
   for (const packageRoot of packageRoots) {
     const scanned = scanIntentPackageAtRoot(packageRoot, {
       fallbackName: parsedUse.packageName,
+      fsCache,
       projectRoot: cwd,
       skillNameHint: parsedUse.skillName,
     })
@@ -225,6 +226,7 @@ function resolveFromPackageRoots(
     if (scanned.package?.name === parsedUse.packageName) {
       const fallbackScanned = scanIntentPackageAtRoot(packageRoot, {
         fallbackName: parsedUse.packageName,
+        fsCache,
         projectRoot: cwd,
       })
       const fallbackResolved = resolveScannedPackageSkill(
@@ -243,6 +245,7 @@ export function resolveSkillUseFastPath(
   options: IntentCoreOptions,
   context = resolveProjectContext({ cwd: process.cwd() }),
   cwd = context.cwd,
+  fsCache = createIntentFsCache(),
 ): ResolveSkillResult | null {
   if (options.globalOnly) return null
   if (shouldSkipFastPathForYarnPnp(context, cwd)) return null
@@ -251,6 +254,7 @@ export function resolveSkillUseFastPath(
     getDirectLoadFastPathCandidateDirs(parsedUse.packageName, context, cwd),
     parsedUse,
     cwd,
+    fsCache,
   )
   if (directResolved) return directResolved
 
@@ -259,8 +263,13 @@ export function resolveSkillUseFastPath(
   }
 
   return resolveFromPackageRoots(
-    getWorkspaceLoadFastPathCandidateDirs(parsedUse.packageName, context),
+    getWorkspaceLoadFastPathCandidateDirs(
+      parsedUse.packageName,
+      context,
+      fsCache,
+    ),
     parsedUse,
     cwd,
+    fsCache,
   )
 }

packages/intent/src/core/types.ts

@@ -1,4 +1,9 @@
-import type { IntentPackage, ScanScope, VersionConflict } from '../types.js'
+import type {
+  IntentPackage,
+  ScanScope,
+  ScanStats,
+  VersionConflict,
+} from '../types.js'
 
 export interface IntentCoreOptions {
   cwd?: string
@@ -60,6 +65,7 @@ export interface IntentSkillListDebug {
   skillCount: number
   warningCount: number
   conflictCount: number
+  scan: IntentScanDebugStats
 }
 
 export interface LoadedIntentSkillDebug {
@@ -73,8 +79,11 @@ export interface LoadedIntentSkillDebug {
   source: IntentPackage['source']
   path: string
   warningCount: number
+  scan: IntentScanDebugStats
 }
 
+export interface IntentScanDebugStats extends ScanStats {}
+
 export type IntentCoreErrorCode =
   | 'invalid-options'
   | 'invalid-skill-use'

packages/intent/src/discovery/walk.ts

@@ -1,15 +1,14 @@
-import { existsSync, readFileSync } from 'node:fs'
+import { existsSync } from 'node:fs'
 import { join } from 'node:path'
 import { listNodeModulesPackageDirs, resolveDepDir, getDeps } from '../utils.js'
-import {
-  readWorkspacePatterns,
-  resolveWorkspacePackages,
-} from '../workspace-patterns.js'
+import { findWorkspacePackages } from '../workspace-patterns.js'
+import type { IntentFsCache } from '../fs-cache.js'
 import type { IntentPackage } from '../types.js'
 
 type PackageJson = Record<string, unknown>
 
 export interface CreateDependencyWalkerOptions {
+  fsCache: IntentFsCache
   projectRoot: string
   readPkgJson: (dirPath: string) => PackageJson | null
   tryRegister: (dirPath: string, fallbackName: string) => boolean
@@ -83,29 +82,26 @@ export function createDependencyWalker(opts: CreateDependencyWalkerOptions) {
     dirPath: string,
     label: string,
   ): PackageJson | null {
-    try {
-      return JSON.parse(
-        readFileSync(join(dirPath, 'package.json'), 'utf8'),
-      ) as PackageJson
-    } catch (err) {
-      const code = (err as NodeJS.ErrnoException).code
+    const result = opts.fsCache.readPackageJsonResult(dirPath)
+    if (!result.packageJson) {
+      const code = (result.error as NodeJS.ErrnoException | null)?.code
       if (code !== 'ENOENT') {
         opts.warnings.push(
-          `Could not read ${label} package.json at ${dirPath}: ${(err as Error).message}`,
+          `Could not read ${label} package.json at ${dirPath}: ${
+            result.error instanceof Error
+              ? result.error.message
+              : 'invalid package.json'
+          }`,
         )
       }
       return null
     }
+
+    return result.packageJson
   }
 
   function walkWorkspacePackages(): void {
-    const workspacePatterns = readWorkspacePatterns(opts.projectRoot)
-    if (!workspacePatterns) return
-
-    for (const wsDir of resolveWorkspacePackages(
-      opts.projectRoot,
-      workspacePatterns,
-    )) {
+    for (const wsDir of findWorkspacePackages(opts.projectRoot)) {
       const wsNodeModules = join(wsDir, 'node_modules')
       if (existsSync(wsNodeModules)) {
         for (const dirPath of listNodeModulesPackageDirs(wsNodeModules)) {