diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index fe450fe..264fcec 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -20,7 +20,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: pnpm - name: Install dependencies diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aebc814..85a2866 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: ${{ fromJSON(github.event_name == 'pull_request' && '["ubuntu-latest","macos-latest","windows-latest"]' || '["ubuntu-latest"]') }} + node-version: [18, 20, 22] steps: - name: Checkout uses: actions/checkout@v4 @@ -26,7 +27,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 20 + node-version: ${{ matrix.node-version }} cache: pnpm - name: Install dependencies diff --git a/.github/workflows/pkg-pr-new-preview.yml b/.github/workflows/pkg-pr-new-preview.yml index 7c96421..22b8d03 100644 --- a/.github/workflows/pkg-pr-new-preview.yml +++ b/.github/workflows/pkg-pr-new-preview.yml @@ -31,7 +31,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: pnpm - name: Install dependencies diff --git a/.gitignore b/.gitignore index da8394e..c2da450 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ docs.lock coverage TODO.md .docs/ +benchmarks/ diff --git a/src/materialize.ts b/src/materialize.ts index 533aadc..b934a30 100644 --- a/src/materialize.ts +++ b/src/materialize.ts @@ -133,11 +133,14 @@ export const materializeSource = async (params: MaterializeParams) => { onlyFiles: true, followSymbolicLinks: false, }); - files.sort((left, right) => - normalizePath(left).localeCompare(normalizePath(right)), - ); + const entries = files + .map((relativePath) => ({ + relativePath, + normalized: normalizePath(relativePath), + })) + .sort((left, right) => left.normalized.localeCompare(right.normalized)); const targetDirs = new Set(); - for (const relativePath of files) { + for (const { relativePath } of entries) { targetDirs.add(path.dirname(relativePath)); } await Promise.all( @@ -149,7 +152,10 @@ export const materializeSource = async (params: MaterializeParams) => { let fileCount = 0; const concurrency = Math.max( 1, - Math.min(files.length, Math.max(8, Math.min(128, os.cpus().length * 8))), + Math.min( + entries.length, + Math.max(8, Math.min(128, os.cpus().length * 8)), + ), ); const manifestPath = path.join(tempDir, MANIFEST_FILENAME); const manifestStream = createWriteStream(manifestPath, { @@ -177,12 +183,11 @@ export const materializeSource = async (params: MaterializeParams) => { }); }; - for (let i = 0; i < files.length; i += concurrency) { - const batch = files.slice(i, i + concurrency); + for (let i = 0; i < entries.length; i += concurrency) { + const batch = entries.slice(i, i + concurrency); const results = await Promise.all( - batch.map(async (relativePath) => { - const relNormalized = normalizePath(relativePath); - const filePath = path.join(params.repoDir, relativePath); + batch.map(async (entry) => { + const filePath = path.join(params.repoDir, entry.relativePath); const fileHandle = await openFileNoFollow(filePath); if (!fileHandle) { return null; @@ -192,7 +197,7 @@ export const materializeSource = async (params: MaterializeParams) => { if (!stats.isFile()) { return null; } - const targetPath = path.join(tempDir, relativePath); + const targetPath = path.join(tempDir, entry.relativePath); ensureSafePath(tempDir, targetPath); if (stats.size >= STREAM_COPY_THRESHOLD_BYTES) { const reader = createReadStream(filePath, { @@ -206,7 +211,7 @@ export const materializeSource = async (params: MaterializeParams) => { await writeFile(targetPath, data); } return { - path: relNormalized, + path: entry.normalized, size: stats.size, }; } finally { diff --git a/src/sync.ts b/src/sync.ts index 8b4097a..228adc3 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -275,6 +275,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { const defaults = plan.defaults; const runFetch = deps.fetchSource ?? fetchSource; const runMaterialize = deps.materializeSource ?? materializeSource; + const docsPresence = new Map(); const buildJobs = async (ids?: string[], force?: boolean) => { const pick = ids?.length ? plan.results.filter((result) => ids.includes(result.id)) @@ -285,9 +286,16 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => { if (!source) { return null; } - const docsPresent = await hasDocs(plan.cacheDir, result.id); + if (force) { + return { result, source }; + } + let docsPresent = docsPresence.get(result.id); + if (docsPresent === undefined) { + docsPresent = await hasDocs(plan.cacheDir, result.id); + docsPresence.set(result.id, docsPresent); + } const needsMaterialize = - force || result.status !== "up-to-date" || !docsPresent; + result.status !== "up-to-date" || !docsPresent; return needsMaterialize ? { result, source } : null; }), ); diff --git a/src/verify.ts b/src/verify.ts index 218e748..4311074 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -46,13 +46,18 @@ export const verifyCache = async (options: VerifyOptions) => { let sizeMismatchCount = 0; for await (const entry of streamManifestEntries(directory)) { const filePath = path.join(directory, entry.path); - if (!(await exists(filePath))) { - missingCount += 1; - continue; - } - const info = await stat(filePath); - if (info.size !== entry.size) { - sizeMismatchCount += 1; + try { + const info = await stat(filePath); + if (info.size !== entry.size) { + sizeMismatchCount += 1; + } + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "ENOTDIR") { + missingCount += 1; + continue; + } + throw error; } } const issues: string[] = []; @@ -74,13 +79,17 @@ export const verifyCache = async (options: VerifyOptions) => { ok: issues.length === 0, issues, }; - } catch (_error) { - return { - ok: false, - issues: [ - label === "source" ? "missing manifest" : "missing target manifest", - ], - }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "ENOTDIR") { + return { + ok: false, + issues: [ + label === "source" ? "missing manifest" : "missing target manifest", + ], + }; + } + throw error; } };