Skip to content

Commit 2493ed8

Browse files
Copiloticlanton
Copilot
and
iclanton
authored
Gate streaming build cache behind useStreamingBuildCache experiment flag
Agent-Logs-Url: https://github.com/microsoft/rushstack/sessions/63fb5abe-e500-4a9c-bd82-3ed613989ef4 Co-authored-by: iclanton <5010588+iclanton@users.noreply.github.com>
1 parent 84a44b7 commit 2493ed8

7 files changed

Lines changed: 61 additions & 14 deletions

File tree

‎common/reviews/api/rush-lib.api.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,7 @@ export interface IExperimentsJson {
490490
usePnpmLockfileOnlyThenFrozenLockfileForRushUpdate?: boolean;
491491
usePnpmPreferFrozenLockfileForRushUpdate?: boolean;
492492
usePnpmSyncForInjectedDependencies?: boolean;
493+
useStreamingBuildCache?: boolean;
493494
}
494495

495496
// @beta
@@ -598,6 +599,7 @@ export interface _IOperationBuildCacheOptions {
598599
buildCacheConfiguration: BuildCacheConfiguration;
599600
excludeAppleDoubleFiles: boolean;
600601
terminal: ITerminal;
602+
useStreamingBuildCache: boolean;
601603
}
602604

603605
// @alpha

‎libraries/rush-lib/src/api/ExperimentsConfiguration.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,14 @@ export interface IExperimentsJson {
136136
* be included in the shared build cache.
137137
*/
138138
omitAppleDoubleFilesFromBuildCache?: boolean;
139+
140+
/**
141+
* If true, the build cache will use streaming APIs to transfer cache entries to and from cloud
142+
* storage. This avoids loading the entire cache entry into memory, which can prevent out-of-memory
143+
* errors for large build outputs. The cloud cache provider plugin must implement the optional
144+
* streaming methods for this to take effect; otherwise it falls back to the buffer-based approach.
145+
*/
146+
useStreamingBuildCache?: boolean;
139147
}
140148

141149
const _EXPERIMENTS_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson);

‎libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,9 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
511511
terminal,
512512
excludeAppleDoubleFiles:
513513
!!this.rushConfiguration.experimentsConfiguration.configuration
514-
.omitAppleDoubleFilesFromBuildCache
514+
.omitAppleDoubleFilesFromBuildCache,
515+
useStreamingBuildCache:
516+
!!this.rushConfiguration.experimentsConfiguration.configuration.useStreamingBuildCache
515517
}).apply(this.hooks);
516518

517519
if (this._debugBuildCacheIdsParameter.value) {

‎libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ export interface IOperationBuildCacheOptions {
3939
* and a companion file exists in the same directory.
4040
*/
4141
excludeAppleDoubleFiles: boolean;
42+
/**
43+
* If true, use streaming APIs (when available) to transfer cache entries to and from the
44+
* cloud provider, avoiding buffering the entire entry in memory.
45+
*/
46+
useStreamingBuildCache: boolean;
4247
}
4348

4449
/**
@@ -82,6 +87,7 @@ export class OperationBuildCache {
8287
private readonly _projectOutputFolderNames: ReadonlyArray<string>;
8388
private readonly _cacheId: string | undefined;
8489
private readonly _excludeAppleDoubleFiles: boolean;
90+
private readonly _useStreamingBuildCache: boolean;
8591

8692
private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) {
8793
const {
@@ -93,7 +99,8 @@ export class OperationBuildCache {
9399
},
94100
project,
95101
projectOutputFolderNames,
96-
excludeAppleDoubleFiles
102+
excludeAppleDoubleFiles,
103+
useStreamingBuildCache
97104
} = options;
98105
this._project = project;
99106
this._localBuildCacheProvider = localCacheProvider;
@@ -103,6 +110,7 @@ export class OperationBuildCache {
103110
this._projectOutputFolderNames = projectOutputFolderNames || [];
104111
this._cacheId = cacheId;
105112
this._excludeAppleDoubleFiles = excludeAppleDoubleFiles && process.platform === 'darwin';
113+
this._useStreamingBuildCache = useStreamingBuildCache;
106114
}
107115

108116
private static _tryGetTarUtility(terminal: ITerminal): Promise<TarExecutable | undefined> {
@@ -126,7 +134,7 @@ export class OperationBuildCache {
126134
executionResult: IOperationExecutionResult,
127135
options: IOperationBuildCacheOptions
128136
): OperationBuildCache {
129-
const { buildCacheConfiguration, terminal, excludeAppleDoubleFiles } = options;
137+
const { buildCacheConfiguration, terminal, excludeAppleDoubleFiles, useStreamingBuildCache } = options;
130138
const outputFolders: string[] = [...(executionResult.operation.settings?.outputFolderNames ?? [])];
131139
if (executionResult.metadataFolderPath) {
132140
outputFolders.push(executionResult.metadataFolderPath);
@@ -139,7 +147,8 @@ export class OperationBuildCache {
139147
phaseName: executionResult.operation.associatedPhase.name,
140148
projectOutputFolderNames: outputFolders,
141149
operationStateHash: executionResult.getStateHash(),
142-
excludeAppleDoubleFiles
150+
excludeAppleDoubleFiles,
151+
useStreamingBuildCache
143152
};
144153
const cacheId: string | undefined = OperationBuildCache._getCacheId(buildCacheOptions);
145154
return new OperationBuildCache(cacheId, buildCacheOptions);
@@ -166,7 +175,10 @@ export class OperationBuildCache {
166175
'This project was not found in the local build cache. Querying the cloud build cache.'
167176
);
168177

169-
if (this._cloudBuildCacheProvider.tryGetCacheEntryStreamByIdAsync) {
178+
if (
179+
this._useStreamingBuildCache &&
180+
this._cloudBuildCacheProvider.tryGetCacheEntryStreamByIdAsync
181+
) {
170182
// Use streaming path to avoid loading the entire cache entry into memory
171183
const cacheEntryStream: Readable | undefined =
172184
await this._cloudBuildCacheProvider.tryGetCacheEntryStreamByIdAsync(terminal, cacheId);
@@ -336,7 +348,10 @@ export class OperationBuildCache {
336348
throw new InternalError('Expected the local cache entry path to be set.');
337349
}
338350

339-
if (this._cloudBuildCacheProvider.trySetCacheEntryStreamAsync) {
351+
if (
352+
this._useStreamingBuildCache &&
353+
this._cloudBuildCacheProvider.trySetCacheEntryStreamAsync
354+
) {
340355
// Use streaming upload to avoid loading the entire cache entry into memory
341356
const entryStream: FileSystemReadStream = FileSystem.createReadStream(localCacheEntryPath);
342357
setCloudCacheEntryPromise = this._cloudBuildCacheProvider.trySetCacheEntryStreamAsync(

‎libraries/rush-lib/src/logic/buildCache/test/OperationBuildCache.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ describe(OperationBuildCache.name, () => {
5959
operationStateHash: '1926f30e8ed24cb47be89aea39e7efd70fcda075',
6060
terminal,
6161
phaseName: 'build',
62-
excludeAppleDoubleFiles: !!options.excludeAppleDoubleFiles
62+
excludeAppleDoubleFiles: !!options.excludeAppleDoubleFiles,
63+
useStreamingBuildCache: false
6364
});
6465

6566
return subject;

‎libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,13 +77,15 @@ export interface ICacheableOperationPluginOptions {
7777
cobuildConfiguration: CobuildConfiguration | undefined;
7878
terminal: ITerminal;
7979
excludeAppleDoubleFiles: boolean;
80+
useStreamingBuildCache: boolean;
8081
}
8182

8283
interface ITryGetOperationBuildCacheOptionsBase<TRecord> {
8384
buildCacheContext: IOperationBuildCacheContext;
8485
buildCacheConfiguration: BuildCacheConfiguration | undefined;
8586
terminal: ITerminal;
8687
excludeAppleDoubleFiles: boolean;
88+
useStreamingBuildCache: boolean;
8789
record: TRecord;
8890
}
8991

@@ -108,7 +110,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
108110
allowWarningsInSuccessfulBuild,
109111
buildCacheConfiguration,
110112
cobuildConfiguration,
111-
excludeAppleDoubleFiles
113+
excludeAppleDoubleFiles,
114+
useStreamingBuildCache
112115
} = this._options;
113116

114117
hooks.beforeExecuteOperations.tap(
@@ -272,7 +275,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
272275
buildCacheConfiguration,
273276
terminal: buildCacheTerminal,
274277
record,
275-
excludeAppleDoubleFiles
278+
excludeAppleDoubleFiles,
279+
useStreamingBuildCache
276280
});
277281

278282
// Try to acquire the cobuild lock
@@ -291,7 +295,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
291295
buildCacheContext,
292296
record,
293297
terminal: buildCacheTerminal,
294-
excludeAppleDoubleFiles
298+
excludeAppleDoubleFiles,
299+
useStreamingBuildCache
295300
});
296301
if (operationBuildCache) {
297302
buildCacheTerminal.writeVerboseLine(
@@ -585,7 +590,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
585590
private _tryGetOperationBuildCache(
586591
options: ITryGetOperationBuildCacheOptions
587592
): OperationBuildCache | undefined {
588-
const { buildCacheConfiguration, buildCacheContext, terminal, record, excludeAppleDoubleFiles } = options;
593+
const {
594+
buildCacheConfiguration,
595+
buildCacheContext,
596+
terminal,
597+
record,
598+
excludeAppleDoubleFiles,
599+
useStreamingBuildCache
600+
} = options;
589601
if (!buildCacheContext.operationBuildCache) {
590602
const { cacheDisabledReason } = buildCacheContext;
591603
if (cacheDisabledReason && !record.operation.settings?.allowCobuildWithoutCache) {
@@ -601,7 +613,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
601613
buildCacheContext.operationBuildCache = OperationBuildCache.forOperation(record, {
602614
buildCacheConfiguration,
603615
terminal,
604-
excludeAppleDoubleFiles
616+
excludeAppleDoubleFiles,
617+
useStreamingBuildCache
605618
});
606619
}
607620

@@ -618,7 +631,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
618631
cobuildConfiguration,
619632
record,
620633
terminal,
621-
excludeAppleDoubleFiles
634+
excludeAppleDoubleFiles,
635+
useStreamingBuildCache
622636
} = options;
623637

624638
if (!buildCacheConfiguration?.buildCacheEnabled) {
@@ -649,7 +663,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
649663
terminal,
650664
operationStateHash,
651665
phaseName: associatedPhase.name,
652-
excludeAppleDoubleFiles
666+
excludeAppleDoubleFiles,
667+
useStreamingBuildCache
653668
});
654669

655670
buildCacheContext.operationBuildCache = operationBuildCache;

‎libraries/rush-lib/src/schemas/experiments.schema.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,10 @@
8585
"omitAppleDoubleFilesFromBuildCache": {
8686
"description": "If true, when running on macOS, Rush will omit AppleDouble files (._*) from build cache archives when a companion file exists in the same directory. AppleDouble files are automatically created by macOS to store extended attributes on filesystems that don't support them, and should generally not be included in the shared build cache.",
8787
"type": "boolean"
88+
},
89+
"useStreamingBuildCache": {
90+
"description": "If true, the build cache will use streaming APIs to transfer cache entries to and from cloud storage. This avoids loading the entire cache entry into memory, which can prevent out-of-memory errors for large build outputs. The cloud cache provider plugin must implement the optional streaming methods for this to take effect; otherwise it falls back to the buffer-based approach.",
91+
"type": "boolean"
8892
}
8993
},
9094
"additionalProperties": false

0 commit comments

Comments
 (0)