Skip to content

Commit fb0341f

Browse files
LPegasusdmichon-msftoctogonz
authored
[rush] Support percentage-based weights for operationSettings in rush-project.json (microsoft#5679)
* feature: Support percentage-based weights for operationSettings in rush-project.json * Resolve CodeReview comments for: 1. libraries/rush-lib/src/logic/operations/WeightedOperationPlugin.ts - Added more comments to explain the percentage weight convert function. - Use `as` syntax to cast error to Error type for better readability. 2. libraries/rush-lib/src/schemas/rush-project.schema.json - Updated the description of the weight property to reflect that the maximum concurrency units is determined by `os.availableParallelism()`. 3. libraries/rush-lib/src/logic/operations/test/WeightedOperationPlugin.test.ts - Fix a test name. * Update libraries/rush-lib/src/schemas/rush-project.schema.json Co-authored-by: David Michon <dmichon@microsoft.com> * Update libraries/rush-lib/src/schemas/rush-project.schema.json Co-authored-by: David Michon <dmichon@microsoft.com> * PR feedback * PR feedback * Prepare to publish a MINOR release of Rush --------- Co-authored-by: LPegasus <lpegasus@users.noreply.github.com> Co-authored-by: David Michon <dmichon@microsoft.com> Co-authored-by: Pete Gonzalez <4673363+octogonz@users.noreply.github.com>
1 parent 009b3d1 commit fb0341f

9 files changed

Lines changed: 292 additions & 32 deletions

File tree

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"changes": [
3+
{
4+
"packageName": "@microsoft/rush",
5+
"comment": "Support percentage weight in operationSettings in rush-project.json file.",
6+
"type": "none"
7+
}
8+
],
9+
"packageName": "@microsoft/rush"
10+
}

common/config/rush/version-policies.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@
103103
"policyName": "rush",
104104
"definitionName": "lockStepVersion",
105105
"version": "5.169.3",
106-
"nextBump": "patch",
106+
"nextBump": "minor",
107107
"mainProject": "@microsoft/rush"
108108
}
109109
]

common/reviews/api/rush-lib.api.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -676,7 +676,7 @@ export interface IOperationSettings {
676676
outputFolderNames?: string[];
677677
parameterNamesToIgnore?: string[];
678678
sharding?: IRushPhaseSharding;
679-
weight?: number;
679+
weight?: number | `${number}%`;
680680
}
681681

682682
// @internal (undocumented)

libraries/rush-lib/src/api/RushProjectConfiguration.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ export interface IOperationSettings {
160160
* How many concurrency units this operation should take up during execution. The maximum concurrent units is
161161
* determined by the -p flag.
162162
*/
163-
weight?: number;
163+
weight?: number | `${number}%`;
164164

165165
/**
166166
* If true, this operation can use cobuilds for orchestration without restoring build cache entries.

libraries/rush-lib/src/cli/parsing/ParseParallelism.ts

Lines changed: 50 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,39 +5,67 @@ import * as os from 'node:os';
55

66
import { IS_WINDOWS } from '../../utilities/executionUtilities';
77

8+
export function getNumberOfCores(): number {
9+
return os.availableParallelism?.() ?? os.cpus().length;
10+
}
11+
12+
/**
13+
* Since the JSON value is a string, it must be a percentage like "50%",
14+
* which we convert to a number based on the available parallelism.
15+
* For example, if the available parallelism (not the -p flag) is 8 and the weight is "50%",
16+
* then the resulting weight will be 4.
17+
*
18+
* @param weight
19+
* @returns
20+
*/
21+
export function parseParallelismPercent(weight: string, numberOfCores: number = getNumberOfCores()): number {
22+
const percentageRegExp: RegExp = /^\d+(\.\d+)?%$/;
23+
24+
if (!percentageRegExp.test(weight)) {
25+
throw new Error(`Expecting a percentage string like "12%" or "34.56%".`);
26+
}
27+
28+
const percentValue: number = parseFloat(weight.slice(0, -1));
29+
30+
if (percentValue <= 0) {
31+
throw new Error(`Invalid percentage value of "${percentValue}": value must be greater than zero`);
32+
}
33+
34+
if (percentValue > 100) {
35+
throw new Error(`Invalid percentage value of "${percentValue}": value must not exceed 100%`);
36+
}
37+
38+
// Use as much CPU as possible, so we round down the weight here
39+
return Math.max(1, Math.floor((percentValue / 100) * numberOfCores));
40+
}
41+
842
/**
943
* Parses a command line specification for desired parallelism.
1044
* Factored out to enable unit tests
1145
*/
1246
export function parseParallelism(
1347
rawParallelism: string | undefined,
14-
numberOfCores: number = os.availableParallelism?.() ?? os.cpus().length
48+
numberOfCores: number = getNumberOfCores()
1549
): number {
1650
if (rawParallelism) {
51+
rawParallelism = rawParallelism.trim();
52+
1753
if (rawParallelism === 'max') {
1854
return numberOfCores;
19-
} else {
20-
const parallelismAsNumber: number = Number(rawParallelism);
21-
22-
if (typeof rawParallelism === 'string' && rawParallelism.trim().endsWith('%')) {
23-
const parsedPercentage: number = Number(rawParallelism.trim().replace(/\%$/, ''));
24-
25-
if (parsedPercentage <= 0 || parsedPercentage > 100) {
26-
throw new Error(
27-
`Invalid percentage value of '${rawParallelism}', value cannot be less than '0%' or more than '100%'`
28-
);
29-
}
30-
31-
const workers: number = Math.floor((parsedPercentage / 100) * numberOfCores);
32-
return Math.max(workers, 1);
33-
} else if (!isNaN(parallelismAsNumber)) {
34-
return Math.max(parallelismAsNumber, 1);
35-
} else {
36-
throw new Error(
37-
`Invalid parallelism value of '${rawParallelism}', expected a number, a percentage, or 'max'`
38-
);
39-
}
4055
}
56+
57+
if (rawParallelism.endsWith('%')) {
58+
return parseParallelismPercent(rawParallelism, numberOfCores);
59+
}
60+
61+
const parallelismAsNumber: number = Number(rawParallelism);
62+
if (!isNaN(parallelismAsNumber)) {
63+
return Math.max(parallelismAsNumber, 1);
64+
}
65+
66+
throw new Error(
67+
`Invalid parallelism value of "${rawParallelism}": expected a number, a percentage string, or "max"`
68+
);
4169
} else {
4270
// If an explicit parallelism number wasn't provided, then choose a sensible
4371
// default.
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Jest Snapshot v1, https://goo.gl/fbAQLP
22

3-
exports[`parseParallelism throwsErrorOnInvalidParallelism 1`] = `"Invalid parallelism value of 'tequila', expected a number, a percentage, or 'max'"`;
3+
exports[`parseParallelism throwsErrorOnInvalidParallelism 1`] = `"Invalid parallelism value of \\"tequila\\": expected a number, a percentage string, or \\"max\\""`;
44

5-
exports[`parseParallelism throwsErrorOnInvalidParallelismPercentage 1`] = `"Invalid percentage value of '200%', value cannot be less than '0%' or more than '100%'"`;
5+
exports[`parseParallelism throwsErrorOnInvalidParallelismPercentage 1`] = `"Invalid percentage value of \\"200\\": value must not exceed 100%"`;

libraries/rush-lib/src/logic/operations/WeightedOperationPlugin.ts

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import type {
1212
import type { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration';
1313
import type { IOperationExecutionResult } from './IOperationExecutionResult';
1414
import type { OperationExecutionRecord } from './OperationExecutionRecord';
15+
import { parseParallelismPercent } from '../../cli/parsing/ParseParallelism';
1516

1617
const PLUGIN_NAME: 'WeightedOperationPlugin' = 'WeightedOperationPlugin';
1718

@@ -41,8 +42,18 @@ function weightOperations(
4142
const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(project);
4243
const operationSettings: IOperationSettings | undefined =
4344
operation.settings ?? projectConfiguration?.operationSettingsByOperationName.get(phase.name);
44-
if (operationSettings?.weight) {
45-
operation.weight = operationSettings.weight;
45+
if (operationSettings?.weight !== undefined) {
46+
if (typeof operationSettings.weight === 'number') {
47+
operation.weight = operationSettings.weight;
48+
} else if (typeof operationSettings.weight === 'string') {
49+
try {
50+
operation.weight = parseParallelismPercent(operationSettings.weight);
51+
} catch (error) {
52+
throw new Error(
53+
`${operation.name} (invalid weight: ${JSON.stringify(operationSettings.weight)}) ${(error as Error).message}`
54+
);
55+
}
56+
}
4657
}
4758
}
4859
Async.validateWeightedIterable(operation);
Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
2+
// See LICENSE in the project root for license information.
3+
4+
import os from 'node:os';
5+
6+
import type { IPhase } from '../../../api/CommandLineConfiguration';
7+
import type { RushConfigurationProject } from '../../../api/RushConfigurationProject';
8+
import type { IOperationSettings, RushProjectConfiguration } from '../../../api/RushProjectConfiguration';
9+
import {
10+
type IExecuteOperationsContext,
11+
PhasedCommandHooks
12+
} from '../../../pluginFramework/PhasedCommandHooks';
13+
import type { IOperationExecutionResult } from '../IOperationExecutionResult';
14+
import { Operation } from '../Operation';
15+
import { WeightedOperationPlugin } from '../WeightedOperationPlugin';
16+
import { MockOperationRunner } from './MockOperationRunner';
17+
18+
const MOCK_PHASE: IPhase = {
19+
name: '_phase:test',
20+
allowWarningsOnSuccess: false,
21+
associatedParameters: new Set(),
22+
dependencies: {
23+
self: new Set(),
24+
upstream: new Set()
25+
},
26+
isSynthetic: false,
27+
logFilenameIdentifier: '_phase_test',
28+
missingScriptBehavior: 'silent'
29+
};
30+
31+
function createProject(packageName: string): RushConfigurationProject {
32+
return {
33+
packageName
34+
} as RushConfigurationProject;
35+
}
36+
37+
function createOperation(options: {
38+
project: RushConfigurationProject;
39+
settings?: IOperationSettings;
40+
isNoOp?: boolean;
41+
}): Operation {
42+
const { project, settings, isNoOp } = options;
43+
return new Operation({
44+
phase: MOCK_PHASE,
45+
project,
46+
settings,
47+
runner: new MockOperationRunner(`${project.packageName} (${MOCK_PHASE.name})`, undefined, false, isNoOp),
48+
logFilenameIdentifier: `${project.packageName}_phase_test`
49+
});
50+
}
51+
52+
function createExecutionRecords(operation: Operation): Map<Operation, IOperationExecutionResult> {
53+
return new Map([
54+
[
55+
operation,
56+
{
57+
operation,
58+
runner: operation.runner
59+
} as unknown as IOperationExecutionResult
60+
]
61+
]);
62+
}
63+
64+
function createContext(
65+
projectConfigurations: ReadonlyMap<RushConfigurationProject, RushProjectConfiguration>,
66+
parallelism: number = os.availableParallelism()
67+
): IExecuteOperationsContext {
68+
return {
69+
projectConfigurations,
70+
parallelism
71+
} as IExecuteOperationsContext;
72+
}
73+
74+
async function applyWeightPluginAsync(
75+
operations: Map<Operation, IOperationExecutionResult>,
76+
context: IExecuteOperationsContext
77+
): Promise<void> {
78+
const hooks: PhasedCommandHooks = new PhasedCommandHooks();
79+
new WeightedOperationPlugin().apply(hooks);
80+
await hooks.beforeExecuteOperations.promise(operations, context);
81+
}
82+
83+
function mockAvailableParallelism(value: number): jest.SpyInstance<number, []> {
84+
return jest.spyOn(os, 'availableParallelism').mockReturnValue(value);
85+
}
86+
87+
describe(WeightedOperationPlugin.name, () => {
88+
afterEach(() => {
89+
jest.restoreAllMocks();
90+
});
91+
92+
it('applies numeric weight from operation settings', async () => {
93+
const project: RushConfigurationProject = createProject('project-number');
94+
const operation: Operation = createOperation({
95+
project,
96+
settings: {
97+
operationName: MOCK_PHASE.name,
98+
weight: 7
99+
}
100+
});
101+
102+
await applyWeightPluginAsync(
103+
createExecutionRecords(operation),
104+
createContext(new Map(), /* Set parallelism to ensure -p does not affect weight calculation */ 1)
105+
);
106+
107+
expect(operation.weight).toBe(7);
108+
});
109+
110+
it('converts percentage weight using available parallelism', async () => {
111+
mockAvailableParallelism(10);
112+
113+
const project: RushConfigurationProject = createProject('project-percent');
114+
const operation: Operation = createOperation({
115+
project,
116+
settings: {
117+
operationName: MOCK_PHASE.name,
118+
weight: '25%'
119+
} as IOperationSettings
120+
});
121+
122+
await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()));
123+
124+
expect(operation.weight).toBe(2);
125+
});
126+
127+
it('reads weight from rush-project configuration when operation settings are undefined', async () => {
128+
mockAvailableParallelism(8);
129+
130+
const project: RushConfigurationProject = createProject('project-config');
131+
const operation: Operation = createOperation({ project });
132+
const projectConfiguration: RushProjectConfiguration = {
133+
operationSettingsByOperationName: new Map([
134+
[
135+
MOCK_PHASE.name,
136+
{
137+
operationName: MOCK_PHASE.name,
138+
weight: '50%'
139+
} as IOperationSettings
140+
]
141+
])
142+
} as unknown as RushProjectConfiguration;
143+
144+
await applyWeightPluginAsync(
145+
createExecutionRecords(operation),
146+
createContext(new Map([[project, projectConfiguration]]))
147+
);
148+
149+
expect(operation.weight).toBe(4);
150+
});
151+
152+
it('use floor when converting percentage weight to avoid zero weight', async () => {
153+
mockAvailableParallelism(16);
154+
155+
const project: RushConfigurationProject = createProject('project-floor');
156+
const operation: Operation = createOperation({
157+
project,
158+
settings: {
159+
operationName: MOCK_PHASE.name,
160+
weight: '33.3333%'
161+
} as IOperationSettings
162+
});
163+
164+
await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()));
165+
166+
expect(operation.weight).toBe(5);
167+
});
168+
169+
it('forces NO-OP operation weight to zero ignore weight settings', async () => {
170+
const project: RushConfigurationProject = createProject('project-no-op');
171+
const operation: Operation = createOperation({
172+
project,
173+
isNoOp: true,
174+
settings: {
175+
operationName: MOCK_PHASE.name,
176+
weight: 100
177+
}
178+
});
179+
180+
await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()));
181+
182+
expect(operation.weight).toBe(0);
183+
});
184+
185+
it('throws for invalid percentage weight format', async () => {
186+
mockAvailableParallelism(16);
187+
188+
const project: RushConfigurationProject = createProject('project-invalid');
189+
const operation: Operation = createOperation({
190+
project,
191+
// @ts-expect-error Testing invalid input
192+
settings: {
193+
operationName: MOCK_PHASE.name,
194+
weight: '12.5a%'
195+
} as IOperationSettings
196+
});
197+
198+
await expect(
199+
applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map()))
200+
).rejects.toThrow(/invalid weight: "12.5a%"/i);
201+
});
202+
});

libraries/rush-lib/src/schemas/rush-project.schema.json

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,18 @@
106106
]
107107
},
108108
"weight": {
109-
"description": "The number of concurrency units that this operation should take up. The maximum concurrency units is determined by the -p flag.",
110-
"type": "integer",
111-
"minimum": 0
109+
"oneOf": [
110+
{
111+
"type": "string",
112+
"pattern": "^[1-9][0-9]*(\\.\\d+)?%$",
113+
"description": "The number of concurrency units that this operation should take up, as a percent of `os.availableParallelism()`. At runtime this value will be clamped to the range `[1, rushParallelism]`, where `rushParallelism` is the requested parallelism to the current Rush command. To have this operation consume no concurrency, use the number 0 instead of a string."
114+
},
115+
{
116+
"description": "The number of concurrency units that this operation should take up. At runtime this value will be clamped to the range `[0, rushParallelism]`, where `rushParallelism` is the requested parallelism to the current Rush command.",
117+
"type": "integer",
118+
"minimum": 0
119+
}
120+
]
112121
},
113122
"allowCobuildWithoutCache": {
114123
"type": "boolean",

0 commit comments

Comments
 (0)