Skip to content

Commit 23cb8b6

Browse files
durable-workflow-opsdurable-workflow-ops
committed
Balance workflow feature CI shards
1 parent a97eb96 commit 23cb8b6

3 files changed

Lines changed: 304 additions & 14 deletions

File tree

‎.github/feature-test-timings.json‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
{
2+
"mysql": {
3+
"tests/Feature/AiWorkflowTest.php": 9.59,
4+
"tests/Feature/AsyncWorkflowTest.php": 5.69,
5+
"tests/Feature/AwaitWithTimeoutWorkflowTest.php": 23.43,
6+
"tests/Feature/AwaitWorkflowTest.php": 15.43,
7+
"tests/Feature/Base64WorkflowTest.php": 15.47,
8+
"tests/Feature/ChildWorkflowSignalingTest.php": 14.26,
9+
"tests/Feature/ConcurrentWorkflowTest.php": 1.85,
10+
"tests/Feature/ContinueAsNewWorkflowTest.php": 4.99,
11+
"tests/Feature/DispatchWorkflowInTransactionTest.php": 4.72,
12+
"tests/Feature/ExceptionLoggingReplayTest.php": 13.27,
13+
"tests/Feature/ExceptionWorkflowTest.php": 14.49,
14+
"tests/Feature/FailingWorkflowTest.php": 4.84,
15+
"tests/Feature/HeartbeatWorkflowTest.php": 11.81,
16+
"tests/Feature/MigrationTest.php": 24.61,
17+
"tests/Feature/NestedSignalRaceConditionTest.php": 6.16,
18+
"tests/Feature/ParentContinueAsNewChildWorkflowTest.php": 5.11,
19+
"tests/Feature/ParentWorkflowTest.php": 19.68,
20+
"tests/Feature/RaceConditionTest.php": 36.15,
21+
"tests/Feature/RetriesWorkflowTest.php": 10.83,
22+
"tests/Feature/SagaChildWorkflowTest.php": 4.02,
23+
"tests/Feature/SagaWorkflowTest.php": 8.71,
24+
"tests/Feature/SideEffectWorkflowTest.php": 5.15,
25+
"tests/Feature/SignalReplayTest.php": 56.78,
26+
"tests/Feature/StateChangedEventTest.php": 1.41,
27+
"tests/Feature/StateMachineWorkflowTest.php": 15.4,
28+
"tests/Feature/TimeoutWorkflowTest.php": 8.09,
29+
"tests/Feature/TimerWorkflowTest.php": 17.9,
30+
"tests/Feature/V2/V2ActivityArgumentCodecTest.php": 3.49,
31+
"tests/Feature/V2/V2ActivityExceptionCodecTest.php": 7.27,
32+
"tests/Feature/V2/V2ActivityOptionsTest.php": 20.77,
33+
"tests/Feature/V2/V2ActivityTaskBridgeTest.php": 37.55,
34+
"tests/Feature/V2/V2ActivityTimeoutTest.php": 28.28,
35+
"tests/Feature/V2/V2ArchiveWorkflowTest.php": 4.58,
36+
"tests/Feature/V2/V2AvroParitySuiteTest.php": 16.37,
37+
"tests/Feature/V2/V2AwaitWorkflowTest.php": 66.49,
38+
"tests/Feature/V2/V2ChildWorkflowNamespaceProjectionTest.php": 2.86,
39+
"tests/Feature/V2/V2CompatibilityWorkflowTest.php": 25.42,
40+
"tests/Feature/V2/V2ConfiguredCoreModelsTest.php": 2.38,
41+
"tests/Feature/V2/V2ContinueAsNewMetadataTest.php": 9.2,
42+
"tests/Feature/V2/V2DeterministicTimeTest.php": 5.38,
43+
"tests/Feature/V2/V2DuplicateStartPolicyTest.php": 21.49,
44+
"tests/Feature/V2/V2EntryMethodTest.php": 21.5,
45+
"tests/Feature/V2/V2GoldenHistoryReplayTest.php": 7.15,
46+
"tests/Feature/V2/V2HistoryTimelineTest.php": 50.06,
47+
"tests/Feature/V2/V2LegacyEventCompatibilityTest.php": 8.59,
48+
"tests/Feature/V2/V2LifecycleEventTest.php": 8.6,
49+
"tests/Feature/V2/V2MemoUpsertTest.php": 15.6,
50+
"tests/Feature/V2/V2MessageCursorContinueAsNewTest.php": 6.11,
51+
"tests/Feature/V2/V2MessageStreamAuthoringTest.php": 7.62,
52+
"tests/Feature/V2/V2NamespaceScopedLoadTest.php": 19.61,
53+
"tests/Feature/V2/V2NonRetryableFailureTest.php": 7.68,
54+
"tests/Feature/V2/V2OperatorMetricsTest.php": 13.63,
55+
"tests/Feature/V2/V2OperatorQueueVisibilityTest.php": 7.07,
56+
"tests/Feature/V2/V2ParentClosePolicyTest.php": 21.93,
57+
"tests/Feature/V2/V2QueryWorkflowTest.php": 90.26,
58+
"tests/Feature/V2/V2RunDetailViewTest.php": 98.12,
59+
"tests/Feature/V2/V2RunSummarySortKeyTest.php": 1.39,
60+
"tests/Feature/V2/V2SagaWorkflowTest.php": 25.95,
61+
"tests/Feature/V2/V2ScheduleTest.php": 148.16,
62+
"tests/Feature/V2/V2SearchAttributeTest.php": 14.07,
63+
"tests/Feature/V2/V2SideEffectWorkflowTest.php": 17.24,
64+
"tests/Feature/V2/V2StartSearchAttributeTest.php": 15.88,
65+
"tests/Feature/V2/V2StructuralLimitTest.php": 98.29,
66+
"tests/Feature/V2/V2TaskDispatchTest.php": 16.26,
67+
"tests/Feature/V2/V2TimerWorkflowTest.php": 18.88,
68+
"tests/Feature/V2/V2UpdateWorkflowTest.php": 74.87,
69+
"tests/Feature/V2/V2UuidWorkflowTest.php": 1.87,
70+
"tests/Feature/V2/V2VersionWorkflowTest.php": 33.25,
71+
"tests/Feature/V2/V2WebhookPollAndControlPlaneTest.php": 21.64,
72+
"tests/Feature/V2/V2WebhookWorkflowTest.php": 216.04,
73+
"tests/Feature/V2/V2WorkflowControlPlaneTest.php": 62.68,
74+
"tests/Feature/V2/V2WorkflowReplayerTest.php": 3.44,
75+
"tests/Feature/V2/V2WorkflowRunRetentionCleanupTest.php": 4.27,
76+
"tests/Feature/V2/V2WorkflowStubFakeTest.php": 23.59,
77+
"tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 112.65,
78+
"tests/Feature/V2/V2WorkflowTest.php": 251.92,
79+
"tests/Feature/V2/V2WorkflowTimeoutTest.php": 26.62,
80+
"tests/Feature/VersionWorkflowTest.php": 18.27,
81+
"tests/Feature/WebhookWorkflowTest.php": 14.85,
82+
"tests/Feature/WorkflowTest.php": 37.51
83+
},
84+
"postgresql": {
85+
"tests/Feature/AiWorkflowTest.php": 8.53,
86+
"tests/Feature/AsyncWorkflowTest.php": 0.99,
87+
"tests/Feature/AwaitWithTimeoutWorkflowTest.php": 20.11,
88+
"tests/Feature/AwaitWorkflowTest.php": 16.37,
89+
"tests/Feature/Base64WorkflowTest.php": 13.42,
90+
"tests/Feature/ChildWorkflowSignalingTest.php": 7.1,
91+
"tests/Feature/ConcurrentWorkflowTest.php": 3.74,
92+
"tests/Feature/ContinueAsNewWorkflowTest.php": 4.0,
93+
"tests/Feature/DispatchWorkflowInTransactionTest.php": 6.64,
94+
"tests/Feature/ExceptionLoggingReplayTest.php": 8.75,
95+
"tests/Feature/ExceptionWorkflowTest.php": 11.18,
96+
"tests/Feature/FailingWorkflowTest.php": 0.86,
97+
"tests/Feature/HeartbeatWorkflowTest.php": 13.77,
98+
"tests/Feature/MigrationTest.php": 5.86,
99+
"tests/Feature/NestedSignalRaceConditionTest.php": 4.57,
100+
"tests/Feature/ParentContinueAsNewChildWorkflowTest.php": 4.06,
101+
"tests/Feature/ParentWorkflowTest.php": 24.41,
102+
"tests/Feature/RaceConditionTest.php": 28.24,
103+
"tests/Feature/RetriesWorkflowTest.php": 6.8,
104+
"tests/Feature/SagaChildWorkflowTest.php": 1.93,
105+
"tests/Feature/SagaWorkflowTest.php": 5.6,
106+
"tests/Feature/SideEffectWorkflowTest.php": 0.82,
107+
"tests/Feature/SignalReplayTest.php": 45.05,
108+
"tests/Feature/StateChangedEventTest.php": 0.41,
109+
"tests/Feature/StateMachineWorkflowTest.php": 13.32,
110+
"tests/Feature/TimeoutWorkflowTest.php": 6.71,
111+
"tests/Feature/TimerWorkflowTest.php": 14.78,
112+
"tests/Feature/V2/V2ActivityArgumentCodecTest.php": 1.4,
113+
"tests/Feature/V2/V2ActivityExceptionCodecTest.php": 2.29,
114+
"tests/Feature/V2/V2ActivityOptionsTest.php": 7.78,
115+
"tests/Feature/V2/V2ActivityTaskBridgeTest.php": 10.6,
116+
"tests/Feature/V2/V2ActivityTimeoutTest.php": 9.01,
117+
"tests/Feature/V2/V2ArchiveWorkflowTest.php": 1.57,
118+
"tests/Feature/V2/V2AvroParitySuiteTest.php": 5.29,
119+
"tests/Feature/V2/V2AwaitWorkflowTest.php": 34.8,
120+
"tests/Feature/V2/V2ChildWorkflowNamespaceProjectionTest.php": 0.84,
121+
"tests/Feature/V2/V2CompatibilityWorkflowTest.php": 8.98,
122+
"tests/Feature/V2/V2ConfiguredCoreModelsTest.php": 3.65,
123+
"tests/Feature/V2/V2ContinueAsNewMetadataTest.php": 4.96,
124+
"tests/Feature/V2/V2DeterministicTimeTest.php": 2.45,
125+
"tests/Feature/V2/V2DuplicateStartPolicyTest.php": 15.32,
126+
"tests/Feature/V2/V2EntryMethodTest.php": 9.84,
127+
"tests/Feature/V2/V2GoldenHistoryReplayTest.php": 2.01,
128+
"tests/Feature/V2/V2HistoryTimelineTest.php": 33.57,
129+
"tests/Feature/V2/V2LegacyEventCompatibilityTest.php": 3.55,
130+
"tests/Feature/V2/V2LifecycleEventTest.php": 3.07,
131+
"tests/Feature/V2/V2MemoUpsertTest.php": 6.3,
132+
"tests/Feature/V2/V2MessageCursorContinueAsNewTest.php": 3.11,
133+
"tests/Feature/V2/V2MessageStreamAuthoringTest.php": 3.63,
134+
"tests/Feature/V2/V2NamespaceScopedLoadTest.php": 5.15,
135+
"tests/Feature/V2/V2NonRetryableFailureTest.php": 3.41,
136+
"tests/Feature/V2/V2OperatorMetricsTest.php": 9.71,
137+
"tests/Feature/V2/V2OperatorQueueVisibilityTest.php": 2.02,
138+
"tests/Feature/V2/V2ParentClosePolicyTest.php": 10.73,
139+
"tests/Feature/V2/V2QueryWorkflowTest.php": 46.81,
140+
"tests/Feature/V2/V2RunDetailViewTest.php": 51.15,
141+
"tests/Feature/V2/V2RunSummarySortKeyTest.php": 0.38,
142+
"tests/Feature/V2/V2SagaWorkflowTest.php": 15.85,
143+
"tests/Feature/V2/V2ScheduleTest.php": 75.55,
144+
"tests/Feature/V2/V2SearchAttributeTest.php": 6.08,
145+
"tests/Feature/V2/V2SideEffectWorkflowTest.php": 9.15,
146+
"tests/Feature/V2/V2StartSearchAttributeTest.php": 6.48,
147+
"tests/Feature/V2/V2StructuralLimitTest.php": 70.58,
148+
"tests/Feature/V2/V2TaskDispatchTest.php": 5.17,
149+
"tests/Feature/V2/V2TimerWorkflowTest.php": 11.7,
150+
"tests/Feature/V2/V2UpdateWorkflowTest.php": 44.44,
151+
"tests/Feature/V2/V2UuidWorkflowTest.php": 0.82,
152+
"tests/Feature/V2/V2VersionWorkflowTest.php": 24.26,
153+
"tests/Feature/V2/V2WebhookPollAndControlPlaneTest.php": 6.34,
154+
"tests/Feature/V2/V2WebhookWorkflowTest.php": 108.24,
155+
"tests/Feature/V2/V2WorkflowControlPlaneTest.php": 26.21,
156+
"tests/Feature/V2/V2WorkflowReplayerTest.php": 4.53,
157+
"tests/Feature/V2/V2WorkflowRunRetentionCleanupTest.php": 1.22,
158+
"tests/Feature/V2/V2WorkflowStubFakeTest.php": 9.83,
159+
"tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 36.3,
160+
"tests/Feature/V2/V2WorkflowTest.php": 138.23,
161+
"tests/Feature/V2/V2WorkflowTimeoutTest.php": 10.24,
162+
"tests/Feature/VersionWorkflowTest.php": 22.98,
163+
"tests/Feature/WebhookWorkflowTest.php": 18.33,
164+
"tests/Feature/WorkflowTest.php": 34.42
165+
}
166+
}

‎.github/workflows/php.yml‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -110,13 +110,7 @@ jobs:
110110
timeout-minutes: 65
111111
run: |
112112
mkdir -p build/test-results
113-
php scripts/ci/split-feature-tests.php \
114-
--dir=tests/Feature \
115-
--shard=${{ matrix.shard }} \
116-
--shards=4 \
117-
--weights=.github/test-weights/feature-mysql.json \
118-
--summary="build/test-results/mysql-shard-${{ matrix.shard }}-summary.txt" \
119-
> /tmp/mysql-shard-files
113+
php scripts/ci/select-feature-shard.php mysql ${{ matrix.shard }} 4 > /tmp/mysql-shard-files
120114
cat /tmp/mysql-shard-files
121115
cp /tmp/mysql-shard-files "build/test-results/mysql-shard-${{ matrix.shard }}-files.txt"
122116
timeout --foreground 60m xargs -a /tmp/mysql-shard-files vendor/bin/phpunit --testdox --testsuite feature --log-junit "build/test-results/mysql-shard-${{ matrix.shard }}.xml"
@@ -208,13 +202,7 @@ jobs:
208202
timeout-minutes: 65
209203
run: |
210204
mkdir -p build/test-results
211-
php scripts/ci/split-feature-tests.php \
212-
--dir=tests/Feature \
213-
--shard=${{ matrix.shard }} \
214-
--shards=2 \
215-
--weights=.github/test-weights/feature-mysql.json \
216-
--summary="build/test-results/pgsql-shard-${{ matrix.shard }}-summary.txt" \
217-
> /tmp/pgsql-shard-files
205+
php scripts/ci/select-feature-shard.php postgresql ${{ matrix.shard }} 2 > /tmp/pgsql-shard-files
218206
cat /tmp/pgsql-shard-files
219207
cp /tmp/pgsql-shard-files "build/test-results/pgsql-shard-${{ matrix.shard }}-files.txt"
220208
timeout --foreground 60m xargs -a /tmp/pgsql-shard-files vendor/bin/phpunit --testdox --testsuite feature --log-junit "build/test-results/pgsql-shard-${{ matrix.shard }}.xml"

‎scripts/ci/select-feature-shard.php‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
if ($argc !== 4) {
6+
fwrite(STDERR, "Usage: php scripts/ci/select-feature-shard.php mysql|postgresql SHARD SHARD_COUNT\n");
7+
exit(2);
8+
}
9+
10+
$database = $argv[1];
11+
$shard = filter_var($argv[2], FILTER_VALIDATE_INT);
12+
$shardCount = filter_var($argv[3], FILTER_VALIDATE_INT);
13+
14+
$hasValidDatabase = in_array($database, ['mysql', 'postgresql'], true);
15+
$hasValidShard = $shard !== false && $shardCount !== false && $shardCount >= 1 && $shard >= 0 && $shard < $shardCount;
16+
17+
if (! $hasValidDatabase || ! $hasValidShard) {
18+
fwrite(STDERR, "Invalid arguments. Expected mysql|postgresql, a zero-based shard, and a positive shard count.\n");
19+
exit(2);
20+
}
21+
22+
$root = dirname(__DIR__, 2);
23+
$timingsPath = $root . '/.github/feature-test-timings.json';
24+
$timings = [];
25+
26+
if (is_file($timingsPath)) {
27+
$decoded = json_decode((string) file_get_contents($timingsPath), true, 512, JSON_THROW_ON_ERROR);
28+
if (is_array($decoded[$database] ?? null)) {
29+
$timings = $decoded[$database];
30+
}
31+
}
32+
33+
$files = discoverFeatureTests($root . '/tests/Feature');
34+
$defaultWeight = defaultWeight($timings);
35+
$weightedFiles = array_map(
36+
static fn (string $file): array => [
37+
'file' => $file,
38+
'weight' => (float) ($timings[$file] ?? $defaultWeight),
39+
],
40+
$files,
41+
);
42+
43+
usort($weightedFiles, static function (array $left, array $right): int {
44+
$byWeight = $right['weight'] <=> $left['weight'];
45+
46+
return $byWeight !== 0 ? $byWeight : strcmp($left['file'], $right['file']);
47+
});
48+
49+
$assignments = array_fill(0, $shardCount, []);
50+
$loads = array_fill(0, $shardCount, 0.0);
51+
52+
foreach ($weightedFiles as $weightedFile) {
53+
$target = 0;
54+
55+
for ($index = 1; $index < $shardCount; $index++) {
56+
$hasLowerLoad = $loads[$index] < $loads[$target];
57+
$hasSameLoadWithFewerFiles = $loads[$index] === $loads[$target]
58+
&& count($assignments[$index]) < count($assignments[$target]);
59+
60+
if ($hasLowerLoad || $hasSameLoadWithFewerFiles) {
61+
$target = $index;
62+
}
63+
}
64+
65+
$assignments[$target][] = $weightedFile['file'];
66+
$loads[$target] += $weightedFile['weight'];
67+
}
68+
69+
$selected = $assignments[$shard];
70+
sort($selected, SORT_STRING);
71+
72+
fwrite(STDERR, sprintf(
73+
"%s feature shard %d/%d selected %d files with %.2fs estimated weight\n",
74+
$database,
75+
$shard,
76+
$shardCount,
77+
count($selected),
78+
$loads[$shard],
79+
));
80+
81+
echo implode(PHP_EOL, $selected);
82+
echo PHP_EOL;
83+
84+
/**
85+
* @return list<string>
86+
*/
87+
function discoverFeatureTests(string $directory): array
88+
{
89+
$files = [];
90+
$directoryIterator = new RecursiveDirectoryIterator($directory, FilesystemIterator::SKIP_DOTS);
91+
$iterator = new RecursiveIteratorIterator($directoryIterator);
92+
93+
foreach ($iterator as $file) {
94+
if (! $file instanceof SplFileInfo || ! $file->isFile() || ! str_ends_with($file->getFilename(), 'Test.php')) {
95+
continue;
96+
}
97+
98+
$files[] = normalizePath($file->getPathname());
99+
}
100+
101+
sort($files, SORT_STRING);
102+
103+
return $files;
104+
}
105+
106+
function normalizePath(string $path): string
107+
{
108+
$path = str_replace('\\', '/', $path);
109+
$marker = '/tests/Feature/';
110+
$position = strpos($path, $marker);
111+
112+
if ($position === false) {
113+
return $path;
114+
}
115+
116+
return 'tests/Feature/' . substr($path, $position + strlen($marker));
117+
}
118+
119+
/**
120+
* @param array<string, mixed> $timings
121+
*/
122+
function defaultWeight(array $timings): float
123+
{
124+
$weights = array_values(array_filter(
125+
array_map(static fn (mixed $weight): float => is_numeric($weight) ? (float) $weight : 0.0, $timings),
126+
static fn (float $weight): bool => $weight > 0.0,
127+
));
128+
129+
if ($weights === []) {
130+
return 10.0;
131+
}
132+
133+
sort($weights, SORT_NUMERIC);
134+
135+
return $weights[(int) floor(count($weights) / 2)];
136+
}

0 commit comments

Comments
 (0)