[rush] Add --debug-build-cache-ids flag (#5173)

* [rush] Add `--debug-build-cache-ids` flag

* Minor cleanup

* Lexicographically sort by operation name

* Sort operations by name for display

* Add comments about hash salt

---------

Co-authored-by: David Michon <dmichon-msft@users.noreply.github.com>

Author: dmichon-msft

common/changes/@microsoft/rush/build-cache-reconfigure_2025-03-27-00-38.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add a new CLI flag `--debug-build-cache-ids` to help with root-causing unexpected cache misses.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@microsoft/rush/build-cache-reconfigure_2025-03-27-18-12.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Sort all operations lexicographically by name for reporting purposes.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/reviews/api/rush-lib.api.md

@@ -585,6 +585,8 @@ export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase {
 export interface IOperationExecutionResult {
     readonly cobuildRunnerId: string | undefined;
     readonly error: Error | undefined;
+    getStateHash(): string;
+    getStateHashComponents(): ReadonlyArray<string>;
     readonly logFilePaths: ILogFilePaths | undefined;
     readonly metadataFolderPath: string | undefined;
     readonly nonCachedDurationMs: number | undefined;

libraries/rush-lib/src/api/BuildCacheConfiguration.ts

@@ -1,7 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
 // See LICENSE in the project root for license information.
 
+import { createHash } from 'node:crypto';
 import * as path from 'path';
+
 import {
   JsonFile,
   JsonSchema,
@@ -17,7 +19,11 @@ import { RushConstants } from '../logic/RushConstants';
 import type { ICloudBuildCacheProvider } from '../logic/buildCache/ICloudBuildCacheProvider';
 import { RushUserConfiguration } from './RushUserConfiguration';
 import { EnvironmentConfiguration } from './EnvironmentConfiguration';
-import { CacheEntryId, type GetCacheEntryIdFunction } from '../logic/buildCache/CacheEntryId';
+import {
+  CacheEntryId,
+  type IGenerateCacheEntryIdOptions,
+  type GetCacheEntryIdFunction
+} from '../logic/buildCache/CacheEntryId';
 import type { CloudBuildCacheProviderFactory, RushSession } from '../pluginFramework/RushSession';
 import schemaJson from '../schemas/build-cache.schema.json';
 
@@ -201,23 +207,41 @@ export class BuildCacheConfiguration {
     );
     const rushUserConfiguration: RushUserConfiguration = await RushUserConfiguration.initializeAsync();
 
-    let getCacheEntryId: GetCacheEntryIdFunction;
+    let innerGetCacheEntryId: GetCacheEntryIdFunction;
     try {
-      getCacheEntryId = CacheEntryId.parsePattern(buildCacheJson.cacheEntryNamePattern);
+      innerGetCacheEntryId = CacheEntryId.parsePattern(buildCacheJson.cacheEntryNamePattern);
     } catch (e) {
       terminal.writeErrorLine(
         `Error parsing cache entry name pattern "${buildCacheJson.cacheEntryNamePattern}": ${e}`
       );
       throw new AlreadyReportedError();
     }
 
+    const { cacheHashSalt = '', cacheProvider } = buildCacheJson;
+    const salt: string = `${RushConstants.buildCacheVersion}${cacheHashSalt ? `${RushConstants.hashDelimiter}${cacheHashSalt}` : ''}`;
+    // Extend the cache entry id with to salt the hash
+    // This facilitates forcing cache invalidation either when the build cache version changes (new version of Rush)
+    // or when the user-side salt changes (need to purge bad cache entries, plugins including additional files)
+    const getCacheEntryId: GetCacheEntryIdFunction = (options: IGenerateCacheEntryIdOptions): string => {
+      const saltedHash: string = createHash('sha1')
+        .update(salt)
+        .update(options.projectStateHash)
+        .digest('hex');
+
+      return innerGetCacheEntryId({
+        phaseName: options.phaseName,
+        projectName: options.projectName,
+        projectStateHash: saltedHash
+      });
+    };
+
     let cloudCacheProvider: ICloudBuildCacheProvider | undefined;
     // Don't configure a cloud cache provider if local-only
-    if (buildCacheJson.cacheProvider !== 'local-only') {
+    if (cacheProvider !== 'local-only') {
       const cloudCacheProviderFactory: CloudBuildCacheProviderFactory | undefined =
-        rushSession.getCloudBuildCacheProviderFactory(buildCacheJson.cacheProvider);
+        rushSession.getCloudBuildCacheProviderFactory(cacheProvider);
       if (!cloudCacheProviderFactory) {
-        throw new Error(`Unexpected cache provider: ${buildCacheJson.cacheProvider}`);
+        throw new Error(`Unexpected cache provider: ${cacheProvider}`);
       }
       cloudCacheProvider = await cloudCacheProviderFactory(buildCacheJson as ICloudBuildCacheJson);
     }

libraries/rush-lib/src/api/test/__snapshots__/RushCommandLine.test.ts.snap

@@ -1260,6 +1260,14 @@ Object {
           "required": false,
           "shortName": undefined,
         },
+        Object {
+          "description": "Logs information about the components of the build cache ids for individual operations. This is useful for debugging the incremental build logic.",
+          "environmentVariable": undefined,
+          "kind": "Flag",
+          "longName": "--debug-build-cache-ids",
+          "required": false,
+          "shortName": undefined,
+        },
         Object {
           "description": "Selects a single instead of the default locale (en-us) for non-ship builds or all locales for ship builds.",
           "environmentVariable": undefined,
@@ -1414,6 +1422,14 @@ Object {
           "required": false,
           "shortName": undefined,
         },
+        Object {
+          "description": "Logs information about the components of the build cache ids for individual operations. This is useful for debugging the incremental build logic.",
+          "environmentVariable": undefined,
+          "kind": "Flag",
+          "longName": "--debug-build-cache-ids",
+          "required": false,
+          "shortName": undefined,
+        },
         Object {
           "description": "Perform a production build, including minification and localization steps",
           "environmentVariable": undefined,
@@ -1555,6 +1571,14 @@ Object {
           "required": false,
           "shortName": undefined,
         },
+        Object {
+          "description": "Logs information about the components of the build cache ids for individual operations. This is useful for debugging the incremental build logic.",
+          "environmentVariable": undefined,
+          "kind": "Flag",
+          "longName": "--debug-build-cache-ids",
+          "required": false,
+          "shortName": undefined,
+        },
         Object {
           "description": "Perform a production build, including minification and localization steps",
           "environmentVariable": undefined,

libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts

@@ -58,6 +58,7 @@ import { WeightedOperationPlugin } from '../../logic/operations/WeightedOperatio
 import { getVariantAsync, VARIANT_PARAMETER } from '../../api/Variants';
 import { Selection } from '../../logic/Selection';
 import { NodeDiagnosticDirPlugin } from '../../logic/operations/NodeDiagnosticDirPlugin';
+import { DebugHashesPlugin } from '../../logic/operations/DebugHashesPlugin';
 
 /**
  * Constructor parameters for PhasedScriptAction.
@@ -79,7 +80,10 @@ export interface IPhasedScriptActionOptions extends IBaseScriptActionOptions<IPh
 }
 
 interface IInitialRunPhasesOptions {
-  executionManagerOptions: Omit<IOperationExecutionManagerOptions, 'beforeExecuteOperations'>;
+  executionManagerOptions: Omit<
+    IOperationExecutionManagerOptions,
+    'beforeExecuteOperations' | 'inputsSnapshot'
+  >;
   initialCreateOperationsContext: ICreateOperationsContext;
   stopwatch: Stopwatch;
   terminal: ITerminal;
@@ -155,6 +159,7 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
   private readonly _variantParameter: CommandLineStringParameter | undefined;
   private readonly _noIPCParameter: CommandLineFlagParameter | undefined;
   private readonly _nodeDiagnosticDirParameter: CommandLineStringParameter;
+  private readonly _debugBuildCacheIdsParameter: CommandLineFlagParameter;
   private readonly _includePhaseDeps: CommandLineFlagParameter | undefined;
 
   public constructor(options: IPhasedScriptActionOptions) {
@@ -186,19 +191,20 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
     new WeightedOperationPlugin().apply(this.hooks);
     new ValidateOperationsPlugin(terminal).apply(this.hooks);
 
-    if (this._enableParallelism) {
-      this._parallelismParameter = this.defineStringParameter({
-        parameterLongName: '--parallelism',
-        parameterShortName: '-p',
-        argumentName: 'COUNT',
-        environmentVariable: EnvironmentVariableNames.RUSH_PARALLELISM,
-        description:
-          'Specifies the maximum number of concurrent processes to launch during a build.' +
-          ' The COUNT should be a positive integer, a percentage value (eg. "50%%") or the word "max"' +
-          ' to specify a count that is equal to the number of CPU cores. If this parameter is omitted,' +
-          ' then the default value depends on the operating system and number of CPU cores.'
-      });
-    }
+    this._parallelismParameter = this._enableParallelism
+      ? this.defineStringParameter({
+          parameterLongName: '--parallelism',
+          parameterShortName: '-p',
+          argumentName: 'COUNT',
+          environmentVariable: EnvironmentVariableNames.RUSH_PARALLELISM,
+          description:
+            'Specifies the maximum number of concurrent processes to launch during a build.' +
+            ' The COUNT should be a positive integer, a percentage value (eg. "50%%") or the word "max"' +
+            ' to specify a count that is equal to the number of CPU cores. If this parameter is omitted,' +
+            ' then the default value depends on the operating system and number of CPU cores.'
+        })
+      : undefined;
+
     this._timelineParameter = this.defineFlagParameter({
       parameterLongName: '--timeline',
       description:
@@ -237,18 +243,18 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
         `Using "--impacted-by A --include-phase-deps" avoids that work by performing "_phase:test" only for downstream projects.`
     });
 
-    if (this._isIncrementalBuildAllowed) {
-      this._changedProjectsOnly = this.defineFlagParameter({
-        parameterLongName: '--changed-projects-only',
-        parameterShortName: '-c',
-        description:
-          'Normally the incremental build logic will rebuild changed projects as well as' +
-          ' any projects that directly or indirectly depend on a changed project. Specify "--changed-projects-only"' +
-          ' to ignore dependent projects, only rebuilding those projects whose files were changed.' +
-          ' Note that this parameter is "unsafe"; it is up to the developer to ensure that the ignored projects' +
-          ' are okay to ignore.'
-      });
-    }
+    this._changedProjectsOnly = this._isIncrementalBuildAllowed
+      ? this.defineFlagParameter({
+          parameterLongName: '--changed-projects-only',
+          parameterShortName: '-c',
+          description:
+            'Normally the incremental build logic will rebuild changed projects as well as' +
+            ' any projects that directly or indirectly depend on a changed project. Specify "--changed-projects-only"' +
+            ' to ignore dependent projects, only rebuilding those projects whose files were changed.' +
+            ' Note that this parameter is "unsafe"; it is up to the developer to ensure that the ignored projects' +
+            ' are okay to ignore.'
+        })
+      : undefined;
 
     this._ignoreHooksParameter = this.defineFlagParameter({
       parameterLongName: '--ignore-hooks',
@@ -257,43 +263,44 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
         'Make sure you know what you are skipping.'
     });
 
-    if (this._watchPhases.size > 0 && !this._alwaysWatch) {
-      // Only define the parameter if it has an effect.
-      this._watchParameter = this.defineFlagParameter({
-        parameterLongName: '--watch',
-        description: `Starts a file watcher after initial execution finishes. Will run the following phases on affected projects: ${Array.from(
-          this._watchPhases,
-          (phase: IPhase) => phase.name
-        ).join(', ')}`
-      });
-    }
+    // Only define the parameter if it has an effect.
+    this._watchParameter =
+      this._watchPhases.size > 0 && !this._alwaysWatch
+        ? this.defineFlagParameter({
+            parameterLongName: '--watch',
+            description: `Starts a file watcher after initial execution finishes. Will run the following phases on affected projects: ${Array.from(
+              this._watchPhases,
+              (phase: IPhase) => phase.name
+            ).join(', ')}`
+          })
+        : undefined;
 
     // If `this._alwaysInstall === undefined`, Rush does not define the parameter
     // but a repository may still define a custom parameter with the same name.
-    if (this._alwaysInstall === false) {
-      this._installParameter = this.defineFlagParameter({
-        parameterLongName: '--install',
-        description:
-          'Normally a phased command expects "rush install" to have been manually run first. If this flag is specified, ' +
-          'Rush will automatically perform an install before processing the current command.'
-      });
-    }
-
-    if (this._alwaysInstall !== undefined) {
-      this._variantParameter = this.defineStringParameter(VARIANT_PARAMETER);
-    }
-
-    if (
+    this._installParameter =
+      this._alwaysInstall === false
+        ? this.defineFlagParameter({
+            parameterLongName: '--install',
+            description:
+              'Normally a phased command expects "rush install" to have been manually run first. If this flag is specified, ' +
+              'Rush will automatically perform an install before processing the current command.'
+          })
+        : undefined;
+
+    this._variantParameter =
+      this._alwaysInstall !== undefined ? this.defineStringParameter(VARIANT_PARAMETER) : undefined;
+
+    const isIpcSupported: boolean =
       this._watchPhases.size > 0 &&
-      this.rushConfiguration.experimentsConfiguration.configuration.useIPCScriptsInWatchMode
-    ) {
-      this._noIPCParameter = this.defineFlagParameter({
-        parameterLongName: '--no-ipc',
-        description:
-          'Disables the IPC feature for the current command (if applicable to selected operations). Operations will not look for a ":ipc" suffixed script.' +
-          'This feature only applies in watch mode and is enabled by default.'
-      });
-    }
+      !!this.rushConfiguration.experimentsConfiguration.configuration.useIPCScriptsInWatchMode;
+    this._noIPCParameter = isIpcSupported
+      ? this.defineFlagParameter({
+          parameterLongName: '--no-ipc',
+          description:
+            'Disables the IPC feature for the current command (if applicable to selected operations). Operations will not look for a ":ipc" suffixed script.' +
+            'This feature only applies in watch mode and is enabled by default.'
+        })
+      : undefined;
 
     this._nodeDiagnosticDirParameter = this.defineStringParameter({
       parameterLongName: '--node-diagnostic-dir',
@@ -303,6 +310,12 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
         'This directory will contain a subdirectory for each project and phase.'
     });
 
+    this._debugBuildCacheIdsParameter = this.defineFlagParameter({
+      parameterLongName: '--debug-build-cache-ids',
+      description:
+        'Logs information about the components of the build cache ids for individual operations. This is useful for debugging the incremental build logic.'
+    });
+
     this.defineScriptParameters();
 
     for (const [{ associatedPhases }, tsCommandLineParameter] of this.customParameters) {
@@ -468,6 +481,10 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
           cobuildConfiguration,
           terminal
         }).apply(this.hooks);
+
+        if (this._debugBuildCacheIdsParameter.value) {
+          new DebugHashesPlugin(terminal).apply(this.hooks);
+        }
       } else if (!this._disableBuildCache) {
         terminal.writeVerboseLine(`Incremental strategy: output preservation`);
         // Explicitly disabling the build cache also disables legacy skip detection.
@@ -525,11 +542,13 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
         projectsInUnknownState: projectSelection
       };
 
-      const executionManagerOptions: Omit<IOperationExecutionManagerOptions, 'beforeExecuteOperations'> = {
+      const executionManagerOptions: Omit<
+        IOperationExecutionManagerOptions,
+        'beforeExecuteOperations' | 'inputsSnapshot'
+      > = {
         quietMode: isQuietMode,
         debugMode: this.parser.isDebug,
         parallelism,
-        changedProjectsOnly,
         beforeExecuteOperationAsync: async (record: OperationExecutionRecord) => {
           return await this.hooks.beforeExecuteOperation.promise(record);
         },
@@ -609,6 +628,7 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
 
     const executionManagerOptions: IOperationExecutionManagerOptions = {
       ...partialExecutionManagerOptions,
+      inputsSnapshot: initialSnapshot,
       beforeExecuteOperationsAsync: async (records: Map<Operation, OperationExecutionRecord>) => {
         await this.hooks.beforeExecuteOperations.promise(records, initialExecuteOperationsContext);
       }
@@ -809,6 +829,7 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
         stopwatch,
         executionManagerOptions: {
           ...executionManagerOptions,
+          inputsSnapshot: state,
           beforeExecuteOperationsAsync: async (records: Map<Operation, OperationExecutionRecord>) => {
             await this.hooks.beforeExecuteOperations.promise(records, executeOperationsContext);
           }