Skip to content

Commit 6945869

Browse files
durable-workflow-opsdurable-workflow-ops
committed
Balance workflow feature CI shards
1 parent 3921faa commit 6945869

3 files changed

Lines changed: 307 additions & 2 deletions

File tree

‎.github/test-weights/feature-mysql.json‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
{
2+
"tests/Feature/AiWorkflowTest.php": 9.592,
3+
"tests/Feature/AsyncWorkflowTest.php": 5.693,
4+
"tests/Feature/AwaitWithTimeoutWorkflowTest.php": 23.435,
5+
"tests/Feature/AwaitWorkflowTest.php": 15.434,
6+
"tests/Feature/Base64WorkflowTest.php": 15.471,
7+
"tests/Feature/ChildWorkflowSignalingTest.php": 14.263,
8+
"tests/Feature/ConcurrentWorkflowTest.php": 1.847,
9+
"tests/Feature/ContinueAsNewWorkflowTest.php": 4.991,
10+
"tests/Feature/DispatchWorkflowInTransactionTest.php": 4.724,
11+
"tests/Feature/ExceptionLoggingReplayTest.php": 13.273,
12+
"tests/Feature/ExceptionWorkflowTest.php": 14.485,
13+
"tests/Feature/FailingWorkflowTest.php": 4.843,
14+
"tests/Feature/HeartbeatWorkflowTest.php": 11.809,
15+
"tests/Feature/MigrationTest.php": 24.61,
16+
"tests/Feature/NestedSignalRaceConditionTest.php": 6.159,
17+
"tests/Feature/ParentContinueAsNewChildWorkflowTest.php": 5.112,
18+
"tests/Feature/ParentWorkflowTest.php": 19.683,
19+
"tests/Feature/RaceConditionTest.php": 36.151,
20+
"tests/Feature/RetriesWorkflowTest.php": 10.827,
21+
"tests/Feature/SagaChildWorkflowTest.php": 4.024,
22+
"tests/Feature/SagaWorkflowTest.php": 8.709,
23+
"tests/Feature/SideEffectWorkflowTest.php": 5.153,
24+
"tests/Feature/SignalReplayTest.php": 56.778,
25+
"tests/Feature/StateChangedEventTest.php": 1.407,
26+
"tests/Feature/StateMachineWorkflowTest.php": 15.398,
27+
"tests/Feature/TimeoutWorkflowTest.php": 8.086,
28+
"tests/Feature/TimerWorkflowTest.php": 17.903,
29+
"tests/Feature/V2/V2ActivityArgumentCodecTest.php": 3.486,
30+
"tests/Feature/V2/V2ActivityExceptionCodecTest.php": 7.272,
31+
"tests/Feature/V2/V2ActivityOptionsTest.php": 20.771,
32+
"tests/Feature/V2/V2ActivityTaskBridgeTest.php": 37.545,
33+
"tests/Feature/V2/V2ActivityTimeoutTest.php": 28.28,
34+
"tests/Feature/V2/V2ArchiveWorkflowTest.php": 4.581,
35+
"tests/Feature/V2/V2AvroParitySuiteTest.php": 16.366,
36+
"tests/Feature/V2/V2AwaitWorkflowTest.php": 66.493,
37+
"tests/Feature/V2/V2ChildWorkflowNamespaceProjectionTest.php": 2.862,
38+
"tests/Feature/V2/V2CompatibilityWorkflowTest.php": 25.417,
39+
"tests/Feature/V2/V2ConfiguredCoreModelsTest.php": 2.375,
40+
"tests/Feature/V2/V2ContinueAsNewMetadataTest.php": 9.199,
41+
"tests/Feature/V2/V2DeterministicTimeTest.php": 5.38,
42+
"tests/Feature/V2/V2DuplicateStartPolicyTest.php": 21.484,
43+
"tests/Feature/V2/V2EntryMethodTest.php": 21.501,
44+
"tests/Feature/V2/V2GoldenHistoryReplayTest.php": 7.152,
45+
"tests/Feature/V2/V2HistoryTimelineTest.php": 50.063,
46+
"tests/Feature/V2/V2LegacyEventCompatibilityTest.php": 8.593,
47+
"tests/Feature/V2/V2LifecycleEventTest.php": 8.596,
48+
"tests/Feature/V2/V2MemoUpsertTest.php": 15.598,
49+
"tests/Feature/V2/V2MessageCursorContinueAsNewTest.php": 6.109,
50+
"tests/Feature/V2/V2MessageStreamAuthoringTest.php": 7.621,
51+
"tests/Feature/V2/V2NamespaceScopedLoadTest.php": 19.611,
52+
"tests/Feature/V2/V2NonRetryableFailureTest.php": 7.681,
53+
"tests/Feature/V2/V2OperatorMetricsTest.php": 13.632,
54+
"tests/Feature/V2/V2OperatorQueueVisibilityTest.php": 7.069,
55+
"tests/Feature/V2/V2ParentClosePolicyTest.php": 21.928,
56+
"tests/Feature/V2/V2QueryWorkflowTest.php": 90.262,
57+
"tests/Feature/V2/V2RunDetailViewTest.php": 98.119,
58+
"tests/Feature/V2/V2RunSummarySortKeyTest.php": 1.386,
59+
"tests/Feature/V2/V2SagaWorkflowTest.php": 25.95,
60+
"tests/Feature/V2/V2ScheduleTest.php": 148.161,
61+
"tests/Feature/V2/V2SearchAttributeTest.php": 14.073,
62+
"tests/Feature/V2/V2SideEffectWorkflowTest.php": 17.24,
63+
"tests/Feature/V2/V2StartSearchAttributeTest.php": 15.877,
64+
"tests/Feature/V2/V2StructuralLimitTest.php": 98.289,
65+
"tests/Feature/V2/V2TaskDispatchTest.php": 16.257,
66+
"tests/Feature/V2/V2TimerWorkflowTest.php": 18.875,
67+
"tests/Feature/V2/V2UpdateWorkflowTest.php": 74.865,
68+
"tests/Feature/V2/V2UuidWorkflowTest.php": 1.87,
69+
"tests/Feature/V2/V2VersionWorkflowTest.php": 33.246,
70+
"tests/Feature/V2/V2WebhookPollAndControlPlaneTest.php": 21.643,
71+
"tests/Feature/V2/V2WebhookWorkflowTest.php": 216.043,
72+
"tests/Feature/V2/V2WorkflowControlPlaneTest.php": 62.68,
73+
"tests/Feature/V2/V2WorkflowReplayerTest.php": 3.437,
74+
"tests/Feature/V2/V2WorkflowRunRetentionCleanupTest.php": 4.271,
75+
"tests/Feature/V2/V2WorkflowStubFakeTest.php": 23.592,
76+
"tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 112.649,
77+
"tests/Feature/V2/V2WorkflowTest.php": 251.922,
78+
"tests/Feature/V2/V2WorkflowTimeoutTest.php": 26.623,
79+
"tests/Feature/VersionWorkflowTest.php": 18.264,
80+
"tests/Feature/WebhookWorkflowTest.php": 14.851,
81+
"tests/Feature/WorkflowTest.php": 37.509
82+
}

‎.github/workflows/php.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,13 @@ jobs:
110110
timeout-minutes: 65
111111
run: |
112112
mkdir -p build/test-results
113-
find tests/Feature -name '*Test.php' | sort | awk "((NR - 1) % 4) == ${{ matrix.shard }}" > /tmp/mysql-shard-files
113+
php scripts/ci/split-feature-tests.php \
114+
--dir=tests/Feature \
115+
--shard=${{ matrix.shard }} \
116+
--shards=4 \
117+
--weights=.github/test-weights/feature-mysql.json \
118+
--summary="build/test-results/mysql-shard-${{ matrix.shard }}-summary.txt" \
119+
> /tmp/mysql-shard-files
114120
cat /tmp/mysql-shard-files
115121
cp /tmp/mysql-shard-files "build/test-results/mysql-shard-${{ matrix.shard }}-files.txt"
116122
timeout --foreground 60m xargs -a /tmp/mysql-shard-files vendor/bin/phpunit --testdox --testsuite feature --log-junit "build/test-results/mysql-shard-${{ matrix.shard }}.xml"
@@ -202,7 +208,13 @@ jobs:
202208
timeout-minutes: 65
203209
run: |
204210
mkdir -p build/test-results
205-
find tests/Feature -name '*Test.php' | sort | awk "((NR - 1) % 2) == ${{ matrix.shard }}" > /tmp/pgsql-shard-files
211+
php scripts/ci/split-feature-tests.php \
212+
--dir=tests/Feature \
213+
--shard=${{ matrix.shard }} \
214+
--shards=2 \
215+
--weights=.github/test-weights/feature-mysql.json \
216+
--summary="build/test-results/pgsql-shard-${{ matrix.shard }}-summary.txt" \
217+
> /tmp/pgsql-shard-files
206218
cat /tmp/pgsql-shard-files
207219
cp /tmp/pgsql-shard-files "build/test-results/pgsql-shard-${{ matrix.shard }}-files.txt"
208220
timeout --foreground 60m xargs -a /tmp/pgsql-shard-files vendor/bin/phpunit --testdox --testsuite feature --log-junit "build/test-results/pgsql-shard-${{ matrix.shard }}.xml"

‎scripts/ci/split-feature-tests.php‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
$options = getopt('', ['dir::', 'shard:', 'shards:', 'weights::', 'summary::']);
6+
7+
$dir = normalize_path((string) ($options['dir'] ?? 'tests/Feature'));
8+
$shard = parse_int_option($options, 'shard');
9+
$shards = parse_int_option($options, 'shards');
10+
$weightsFile = isset($options['weights']) ? (string) $options['weights'] : null;
11+
$summaryFile = isset($options['summary']) ? (string) $options['summary'] : null;
12+
13+
if ($shards < 1) {
14+
fail('--shards must be greater than zero.');
15+
}
16+
17+
if ($shard < 0 || $shard >= $shards) {
18+
fail('--shard must be between 0 and --shards minus one.');
19+
}
20+
21+
if (! is_dir($dir)) {
22+
fail("Test directory [{$dir}] does not exist.");
23+
}
24+
25+
$weights = $weightsFile !== null ? load_weights($weightsFile) : [];
26+
$files = find_test_files($dir);
27+
28+
if ($files === []) {
29+
fail("No test files found under [{$dir}].");
30+
}
31+
32+
$assignments = assign_weighted_shards($files, $weights, $shards);
33+
$selected = $assignments[$shard]['files'];
34+
sort($selected, SORT_STRING);
35+
36+
foreach ($selected as $file) {
37+
echo $file . PHP_EOL;
38+
}
39+
40+
if ($summaryFile !== null) {
41+
write_summary($summaryFile, $assignments, $weights);
42+
}
43+
44+
/**
45+
* @param array<string, mixed> $options
46+
*/
47+
function parse_int_option(array $options, string $name): int
48+
{
49+
if (! isset($options[$name]) || ! is_scalar($options[$name]) || $options[$name] === '') {
50+
fail("--{$name} is required.");
51+
}
52+
53+
if (! preg_match('/^\d+$/', (string) $options[$name])) {
54+
fail("--{$name} must be a non-negative integer.");
55+
}
56+
57+
return (int) $options[$name];
58+
}
59+
60+
function normalize_path(string $path): string
61+
{
62+
return str_replace('\\', '/', rtrim($path, '/'));
63+
}
64+
65+
/**
66+
* @return array<string, float>
67+
*/
68+
function load_weights(string $path): array
69+
{
70+
if (! is_file($path)) {
71+
fail("Weights file [{$path}] does not exist.");
72+
}
73+
74+
$decoded = json_decode((string) file_get_contents($path), true);
75+
76+
if (! is_array($decoded)) {
77+
fail("Weights file [{$path}] must contain a JSON object.");
78+
}
79+
80+
$weights = [];
81+
82+
foreach ($decoded as $file => $weight) {
83+
if (! is_string($file) || (! is_int($weight) && ! is_float($weight))) {
84+
fail("Weights file [{$path}] must map test file paths to numeric weights.");
85+
}
86+
87+
if ($weight <= 0) {
88+
fail("Weight for [{$file}] must be greater than zero.");
89+
}
90+
91+
$weights[normalize_path($file)] = (float) $weight;
92+
}
93+
94+
return $weights;
95+
}
96+
97+
/**
98+
* @return list<string>
99+
*/
100+
function find_test_files(string $dir): array
101+
{
102+
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS));
103+
104+
$files = [];
105+
106+
foreach ($iterator as $file) {
107+
if (! $file instanceof SplFileInfo || ! $file->isFile()) {
108+
continue;
109+
}
110+
111+
$path = normalize_path($file->getPathname());
112+
113+
if (str_ends_with($path, 'Test.php')) {
114+
$files[] = $path;
115+
}
116+
}
117+
118+
sort($files, SORT_STRING);
119+
120+
return $files;
121+
}
122+
123+
/**
124+
* @param list<string> $files
125+
* @param array<string, float> $weights
126+
* @return list<array{files: list<string>, weight: float}>
127+
*/
128+
function assign_weighted_shards(array $files, array $weights, int $shards): array
129+
{
130+
$weightedFiles = [];
131+
132+
foreach ($files as $file) {
133+
$weightedFiles[] = [
134+
'file' => $file,
135+
'weight' => $weights[$file] ?? 1.0,
136+
];
137+
}
138+
139+
usort(
140+
$weightedFiles,
141+
static fn (array $a, array $b): int => ($b['weight'] <=> $a['weight'])
142+
?: strcmp($a['file'], $b['file'])
143+
);
144+
145+
$assignments = array_fill(0, $shards, null);
146+
147+
for ($i = 0; $i < $shards; $i++) {
148+
$assignments[$i] = [
149+
'files' => [],
150+
'weight' => 0.0,
151+
];
152+
}
153+
154+
foreach ($weightedFiles as $weightedFile) {
155+
$target = 0;
156+
157+
for ($i = 1; $i < $shards; $i++) {
158+
if ($assignments[$i]['weight'] < $assignments[$target]['weight']) {
159+
$target = $i;
160+
161+
continue;
162+
}
163+
164+
if (
165+
$assignments[$i]['weight'] === $assignments[$target]['weight']
166+
&& count($assignments[$i]['files']) < count($assignments[$target]['files'])
167+
) {
168+
$target = $i;
169+
}
170+
}
171+
172+
$assignments[$target]['files'][] = $weightedFile['file'];
173+
$assignments[$target]['weight'] += $weightedFile['weight'];
174+
}
175+
176+
return $assignments;
177+
}
178+
179+
/**
180+
* @param list<array{files: list<string>, weight: float}> $assignments
181+
* @param array<string, float> $weights
182+
*/
183+
function write_summary(string $path, array $assignments, array $weights): void
184+
{
185+
$rows = [];
186+
187+
foreach ($assignments as $index => $assignment) {
188+
$rows[] = sprintf(
189+
'shard=%d files=%d estimated_weight=%.3f',
190+
$index,
191+
count($assignment['files']),
192+
$assignment['weight']
193+
);
194+
195+
$files = $assignment['files'];
196+
sort($files, SORT_STRING);
197+
198+
foreach ($files as $file) {
199+
$rows[] = sprintf(' %.3f %s', $weights[$file] ?? 1.0, $file);
200+
}
201+
}
202+
203+
file_put_contents($path, implode(PHP_EOL, $rows) . PHP_EOL);
204+
}
205+
206+
function fail(string $message): never
207+
{
208+
fwrite(STDERR, $message . PHP_EOL);
209+
210+
exit(1);
211+
}

0 commit comments

Comments
 (0)