Skip to content

Commit ef98ade

Browse files
authored
fix(capsule): prune by age only, drop size-based eviction (#10408)
## Problem The capsule auto-prune (`bit capsule prune --size-target 10`) was wiping scope-aspects capsules across **all** repos sharing the global cache. With ~10 bit checkouts, every aspect capsule got evicted daily, making rebuilds slow. Root cause is in `applySizeTarget`: it measured the **whole** cache total but could only evict scope-aspect children. When the un-evictable remainder (workspace/scope caps, ~11 GB of legacy unmarked dirs, each aspect-root's own `node_modules`) already exceeded the target, the loop evicted 100% of aspect capsules across every repo and still never reached the target. Evidence from logs: 569 removals tagged `size-target-10gb` vs only 14 for `older-than-30d`, and the cache stayed at 18 GB after "pruning" to a 10 GB target. Size accounting also forced an expensive recursive `lstat` walk of the entire cache on every auto-prune. ## Change Drop size-based eviction entirely; prune by **age + orphan** only. - Auto-prune spawns `bit capsule prune --older-than <N>` (no `--size-target`); no size walk → O(1) rename-to-trash deletes. - Removed `applySizeTarget`, the `--size-target` flag, and `CFG_CAPSULES_MAX_SIZE_GB`. - `--with-sizes` stays as an opt-in for byte totals in the report. The cache now self-bounds to "workspaces touched within `--older-than` (default 30d)"; throwaway/temp scopes are still caught by the orphan check.
1 parent 5b56dfb commit ef98ade

4 files changed

Lines changed: 15 additions & 105 deletions

File tree

components/legacy/constants/constants.ts

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -393,12 +393,6 @@ export const CFG_USE_DATED_CAPSULES = 'use_dated_capsules';
393393

394394
export const CFG_CACHE_LOCK_ONLY_CAPSULES = 'cache_lock_only_capsules';
395395

396-
/**
397-
* Soft size limit (in GB) for the global capsules dir. When exceeded, the auto-prune trigger
398-
* runs an LRU eviction of aspect-version subdirs until the cache drops below this size.
399-
*/
400-
export const CFG_CAPSULES_MAX_SIZE_GB = 'capsules_max_size_gb';
401-
402396
/**
403397
* Age threshold (in days) for aspect-version and scope capsule pruning. Capsules whose
404398
* last-used marker is older than this are considered stale and removed. Workspace capsules

scopes/component/isolator/capsule-cache.ts

Lines changed: 14 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import { isFeatureEnabled, CAPSULE_AUTO_PRUNE } from '@teambit/harmony.modules.f
2121
import {
2222
CFG_CAPSULES_AUTO_PRUNE,
2323
CFG_CAPSULES_MAX_AGE_DAYS,
24-
CFG_CAPSULES_MAX_SIZE_GB,
2524
CFG_CAPSULES_SCOPES_ASPECTS_DATED_DIR,
2625
} from '@teambit/legacy.constants';
2726
import type CapsuleList from './capsule-list';
@@ -49,13 +48,12 @@ export type PruneCapsulesOptions = {
4948
olderThanDays?: number;
5049
includeOrphans?: boolean;
5150
keepWorkspaceCaps?: boolean;
52-
sizeTargetGb?: number;
5351
dryRun?: boolean;
5452
/**
5553
* Compute byte sizes for every entry being considered. When false, all `sizeBytes`
5654
* in the report are 0 and the cache walk skips the expensive recursive `lstat` pass —
5755
* deletion (rename-to-trash) is O(1) and runs in milliseconds even on multi-GB caches.
58-
* Forced on when `sizeTargetGb` is set because that path needs sizes to enforce.
56+
* Opt-in (via `bit capsule prune --with-sizes`) since the walk is slow on large caches.
5957
*/
6058
withSizes?: boolean;
6159
};
@@ -257,14 +255,11 @@ export class CapsuleCache {
257255
await fs.outputFile(stampPath, '');
258256

259257
// Guard against non-numeric/empty config (NaN) and negative values (which would
260-
// invert the age cutoff / size target and wipe the whole cache).
258+
// invert the age cutoff and wipe the whole cache).
261259
const olderThanDays = Math.max(0, toFiniteNumber(this.configStore.getConfig(CFG_CAPSULES_MAX_AGE_DAYS)) ?? 30);
262-
const sizeTargetGb = Math.max(0, toFiniteNumber(this.configStore.getConfig(CFG_CAPSULES_MAX_SIZE_GB)) ?? 10);
263260

264-
this.logger.debug(
265-
`[auto-prune] spawning detached child. olderThanDays=${olderThanDays}, sizeTargetGb=${sizeTargetGb}`
266-
);
267-
this.spawnDetachedAutoPrune(olderThanDays, sizeTargetGb);
261+
this.logger.debug(`[auto-prune] spawning detached child. olderThanDays=${olderThanDays}`);
262+
this.spawnDetachedAutoPrune(olderThanDays);
268263
} finally {
269264
if (claimed) {
270265
try {
@@ -284,22 +279,18 @@ export class CapsuleCache {
284279
* Recursion guard: the child also runs onBeforeExit → maybeAutoPrune, but it reads the
285280
* stamp file that we just wrote and bails out before re-spawning.
286281
*/
287-
private spawnDetachedAutoPrune(olderThanDays: number, sizeTargetGb: number): void {
282+
private spawnDetachedAutoPrune(olderThanDays: number): void {
288283
const bitEntry = process.argv[1];
289284
if (!bitEntry) {
290285
this.logger.debug('[auto-prune] cannot detach: process.argv[1] is empty');
291286
return;
292287
}
293288
try {
294-
const child = spawn(
295-
process.execPath,
296-
[bitEntry, 'capsule', 'prune', '--older-than', String(olderThanDays), '--size-target', String(sizeTargetGb)],
297-
{
298-
detached: true,
299-
stdio: 'ignore',
300-
windowsHide: true,
301-
}
302-
);
289+
const child = spawn(process.execPath, [bitEntry, 'capsule', 'prune', '--older-than', String(olderThanDays)], {
290+
detached: true,
291+
stdio: 'ignore',
292+
windowsHide: true,
293+
});
303294
child.unref();
304295
} catch (err: any) {
305296
this.logger.debug(`[auto-prune] failed to spawn detached child: ${err.message}`);
@@ -507,21 +498,18 @@ export class CapsuleCache {
507498
* - scope-aspects-root: never deleted as a whole; per-aspect-version children pruned by age
508499
* - scope caps and unmarked dirs older than threshold: deleted
509500
* - orphans (marker says originPath gone): deleted
510-
* - after the above, if sizeTargetGb given and size still exceeds it, evict oldest-first
511501
*/
512502
async pruneCapsules(opts: PruneCapsulesOptions = {}): Promise<PruneCapsulesReport> {
513503
// Clamp to >= 0: a negative age would put the cutoff in the future (everything looks
514-
// "too old" → whole cache deleted); a negative size target would force evicting
515-
// everything. Both are almost certainly user error, so floor them at 0.
504+
// "too old" → whole cache deleted), almost certainly user error, so floor it at 0.
516505
const olderThanDays = Math.max(0, opts.olderThanDays ?? 30);
517-
const sizeTargetGb = opts.sizeTargetGb === undefined ? undefined : Math.max(0, opts.sizeTargetGb);
518506
const includeOrphans = opts.includeOrphans !== false;
519507
const keepWorkspaceCaps = opts.keepWorkspaceCaps === true;
520508
const dryRun = opts.dryRun === true;
521509
// Size accounting requires an expensive recursive lstat across the whole cache. Skip
522-
// it by default so the foreground command returns in ms (deletes are O(1) renames);
523-
// force on for size-target enforcement and when the caller asks for byte accounting.
524-
const computeSizes = opts.withSizes === true || sizeTargetGb !== undefined;
510+
// it by default so the command returns in ms (deletes are O(1) renames); opt in via
511+
// `--with-sizes` when you want byte totals in the report.
512+
const computeSizes = opts.withSizes === true;
525513
const ageCutoffMs = Date.now() - olderThanDays * ONE_DAY_MS;
526514
const datedDirName = this.configStore.getConfig(CFG_CAPSULES_SCOPES_ASPECTS_DATED_DIR) || 'dated-capsules';
527515

@@ -568,10 +556,6 @@ export class CapsuleCache {
568556
}
569557
}
570558

571-
if (sizeTargetGb !== undefined) {
572-
await this.applySizeTarget(sizeTargetGb, removed, dryRun);
573-
}
574-
575559
const totalRemovedBytes = removed.reduce((sum, r) => sum + r.sizeBytes, 0);
576560
// For dry-run, report the *projected* post-prune size so the CLI summary stays
577561
// internally consistent (cache: X → X − freed). Real prune subtracts the same.
@@ -692,63 +676,4 @@ export class CapsuleCache {
692676
}
693677
}
694678
}
695-
696-
/**
697-
* After the standard prune, if total still exceeds the target, keep evicting the
698-
* oldest remaining aspect-version subdirs until under the limit.
699-
*/
700-
private async applySizeTarget(
701-
sizeTargetGb: number,
702-
removed: PruneCapsulesReport['removed'],
703-
dryRun: boolean
704-
): Promise<void> {
705-
const targetBytes = sizeTargetGb * 1024 * 1024 * 1024;
706-
const removedPaths = new Set(removed.map((r) => r.path));
707-
// Re-walk what's left (one pass, with sizes) to find both the current total and the
708-
// oldest aspect-version children to evict.
709-
const roots = await this.listAllCapsuleRoots();
710-
const totalBytes = roots.reduce((sum, r) => sum + r.sizeBytes, 0);
711-
const aspectChildren: Array<{ path: string; lastUsedMs: number; sizeBytes: number; originPath?: string }> = [];
712-
for (const root of roots) {
713-
if (
714-
root.kind !== 'scope-aspects-root' &&
715-
!(root.kind === 'unmarked' && (await this.looksLikeAspectsRoot(root.path)))
716-
) {
717-
continue;
718-
}
719-
let entries: fs.Dirent[] = [];
720-
try {
721-
entries = await fs.readdir(root.path, { withFileTypes: true });
722-
} catch {
723-
continue;
724-
}
725-
for (const entry of entries) {
726-
if (!this.isPrunableSubdir(entry)) continue;
727-
const childPath = path.join(root.path, entry.name);
728-
if (removedPaths.has(childPath)) continue;
729-
const { marker, lastUsedMs } = await this.readMarkerInfo(childPath);
730-
const sizeBytes = await this.computeDirSize(childPath);
731-
aspectChildren.push({ path: childPath, lastUsedMs, sizeBytes, originPath: marker?.originPath });
732-
}
733-
}
734-
aspectChildren.sort((a, b) => a.lastUsedMs - b.lastUsedMs);
735-
// In dry-run the standard-prune entries are still on disk (counted in totalBytes), so
736-
// subtract them; in a real run they were already moved to trash and excluded from the walk.
737-
let remainingBytes = totalBytes - (dryRun ? removed.reduce((s, r) => s + r.sizeBytes, 0) : 0);
738-
for (const child of aspectChildren) {
739-
if (remainingBytes <= targetBytes) break;
740-
await this.recordRemoval(
741-
removed,
742-
{
743-
path: child.path,
744-
kind: 'scope-aspect',
745-
reason: `size-target-${sizeTargetGb}gb`,
746-
sizeBytes: child.sizeBytes,
747-
originPath: child.originPath,
748-
},
749-
dryRun
750-
);
751-
remainingBytes -= child.sizeBytes;
752-
}
753-
}
754679
}

scopes/harmony/cli-reference/cli-reference.mdx

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,6 @@ use --dry-run first to preview what would be removed.
313313
| `--older-than <days>` | | age threshold in days for aspect-version/scope capsule pruning (default 30) |
314314
| `--keep-workspace-caps` | | skip workspace capsule deletion |
315315
| `--no-orphans` | | don't delete capsules whose origin path no longer exists |
316-
| `--size-target <gb>` | | after standard pruning, LRU-evict aspect-versions until total drops below this size (forces --with-sizes) |
317316
| `--dry-run` | | preview what would be removed without deleting |
318317
| `--with-sizes` | | compute byte sizes for the report (walks the full cache; slow on large caches). default: off — sizes shown as 0 but the prune itself is instant. |
319318
| `--json` | `-j` | json format |

scopes/workspace/workspace/capsule.cmd.ts

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import type { CapsuleList, IsolateComponentsOptions, IsolatorMain, PruneCapsules
55
import { CAPSULE_ORIGIN_FILE } from '@teambit/isolator';
66
import type { ScopeMain } from '@teambit/scope';
77
import type { ConfigStoreMain } from '@teambit/config-store';
8-
import { CFG_CAPSULES_MAX_AGE_DAYS, CFG_CAPSULES_MAX_SIZE_GB } from '@teambit/legacy.constants';
8+
import { CFG_CAPSULES_MAX_AGE_DAYS } from '@teambit/legacy.constants';
99
import chalk from 'chalk';
1010
import fs from 'fs-extra';
1111
import path from 'path';
@@ -288,7 +288,6 @@ type PruneOpts = {
288288
olderThan?: number;
289289
keepWorkspaceCaps?: boolean;
290290
noOrphans?: boolean;
291-
sizeTarget?: number;
292291
dryRun?: boolean;
293292
withSizes?: boolean;
294293
json?: boolean;
@@ -305,11 +304,6 @@ use --dry-run first to preview what would be removed.`;
305304
['', 'older-than <days>', 'age threshold in days for aspect-version/scope capsule pruning (default 30)'],
306305
['', 'keep-workspace-caps', 'skip workspace capsule deletion'],
307306
['', 'no-orphans', "don't delete capsules whose origin path no longer exists"],
308-
[
309-
'',
310-
'size-target <gb>',
311-
'after standard pruning, LRU-evict aspect-versions until total drops below this size (forces --with-sizes)',
312-
],
313307
['', 'dry-run', 'preview what would be removed without deleting'],
314308
[
315309
'',
@@ -362,12 +356,10 @@ use --dry-run first to preview what would be removed.`;
362356
// CLI flags win; otherwise fall back to user config so manual and auto-prune
363357
// behave consistently. Final fallback to defaults happens in `pruneCapsules`.
364358
const olderThanFromConfig = this.configStore.getConfig(CFG_CAPSULES_MAX_AGE_DAYS);
365-
const sizeTargetFromConfig = this.configStore.getConfig(CFG_CAPSULES_MAX_SIZE_GB);
366359
return this.isolator.pruneCapsules({
367360
olderThanDays: toFiniteNumber(opts.olderThan) ?? toFiniteNumber(olderThanFromConfig),
368361
keepWorkspaceCaps: opts.keepWorkspaceCaps === true,
369362
includeOrphans: opts.noOrphans !== true,
370-
sizeTargetGb: toFiniteNumber(opts.sizeTarget) ?? toFiniteNumber(sizeTargetFromConfig),
371363
dryRun: opts.dryRun === true,
372364
withSizes: opts.withSizes === true,
373365
});

0 commit comments

Comments
 (0)