Skip to content

Commit 64bb215

Browse files
xHeavenxHeaven
authored
fix(ci): run benchmark when PR goes to base branch (#1992)
1 parent f532ddc commit 64bb215

4 files changed

Lines changed: 297 additions & 12 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@ name: Benchmark
22

33
on:
44
pull_request:
5-
branches: [main]
65

76
concurrency:
87
group: benchmark-${{ github.event.pull_request.number }}
98
cancel-in-progress: true
109

1110
jobs:
1211
benchmark:
12+
if: ${{ github.event.pull_request.base.ref == github.event.repository.default_branch }}
1313
name: Performance Regression Check
1414
runs-on: ubuntu-latest
1515

@@ -22,7 +22,7 @@ jobs:
2222
- name: Setup PHP
2323
uses: shivammathur/setup-php@v2
2424
with:
25-
php-version: 8.4
25+
php-version: 8.5
2626
extensions: dom, curl, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, fileinfo
2727
coverage: none
2828

@@ -32,14 +32,20 @@ jobs:
3232
- name: Benchmark base branch
3333
run: |
3434
git checkout ${{ github.event.pull_request.base.sha }}
35+
cp .env.example .env
36+
sed -i 's/^INTERNAL_CACHES=.*/INTERNAL_CACHES=false/' .env
37+
sed -i 's/^DISCOVERY_CACHE=.*/DISCOVERY_CACHE=false/' .env
3538
composer install --no-interaction --prefer-dist --quiet
3639
vendor/bin/phpbench run --tag=base --store --progress=none
3740
3841
- name: Benchmark PR branch
3942
run: |
4043
git checkout ${{ github.event.pull_request.head.sha }}
44+
cp .env.example .env
45+
sed -i 's/^INTERNAL_CACHES=.*/INTERNAL_CACHES=false/' .env
46+
sed -i 's/^DISCOVERY_CACHE=.*/DISCOVERY_CACHE=false/' .env
4147
composer install --no-interaction --prefer-dist --quiet
42-
vendor/bin/phpbench run --tag=pr --store --ref=base --report=aggregate --progress=none --output=markdown | tee benchmark-result.txt || true
48+
vendor/bin/phpbench run --tag=pr --store --ref=base --report=aggregate --progress=none --output=benchmark-comment | tee benchmark-result.txt || true
4349
4450
- name: Prepare artifact
4551
run: |

‎phpbench.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,13 @@
33
"runner.bootstrap": "vendor/autoload.php",
44
"runner.path": "tests/Benchmark",
55
"runner.file_pattern": "*Bench.php",
6+
"runner.retry_threshold": 5,
7+
"report.outputs": {
8+
"benchmark-comment": {
9+
"extends": "markdown",
10+
"outlier_min_diff": 5.0
11+
}
12+
},
613
"core.extensions": [
714
"Tests\\Tempest\\Benchmark\\Extension\\MarkdownExtension"
815
]

‎tests/Benchmark/Extension/MarkdownRenderer.php‎

Lines changed: 109 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,20 @@
1717

1818
final readonly class MarkdownRenderer implements RendererInterface
1919
{
20+
private const array COMPACT_HEADERS = ['Benchmark', 'Set', 'Mem. Peak', 'Time', 'Variability'];
21+
22+
private const array COMPACT_SOURCE_COLUMNS = ['benchmark', 'subject', 'set', 'mem_peak', 'mode', 'rstdev'];
23+
24+
private const int COMPACT_TIME_COLUMN_INDEX = 3;
25+
2026
public function __construct(
2127
private OutputInterface $output,
2228
private Printer $printer,
2329
) {}
2430

2531
public function render(Reports $reports, Config $config): void
2632
{
27-
$content = $this->renderContent($reports);
33+
$content = $this->renderContent($reports, $this->resolveOutlierMinDiff($config));
2834
$file = $config['file'];
2935

3036
if ($file === null) {
@@ -40,22 +46,24 @@ public function configure(OptionsResolver $options): void
4046
{
4147
$options->setDefaults([
4248
'file' => null,
49+
'outlier_min_diff' => null,
4350
]);
4451
$options->setAllowedTypes('file', ['null', 'string']);
52+
$options->setAllowedTypes('outlier_min_diff', ['null', 'float', 'int']);
4553
}
4654

47-
private function renderContent(Reports $reports): string
55+
private function renderContent(Reports $reports, ?float $outlierMinDiff): string
4856
{
4957
$lines = [];
5058

5159
foreach ($reports->tables() as $table) {
52-
array_push($lines, ...$this->renderTable($table));
60+
array_push($lines, ...$this->renderTable($table, $outlierMinDiff));
5361
}
5462

5563
return implode("\n", $lines) . "\n";
5664
}
5765

58-
private function renderTable(Table $table): array
66+
private function renderTable(Table $table, ?float $outlierMinDiff): array
5967
{
6068
$lines = [];
6169
$title = $table->title();
@@ -71,11 +79,22 @@ private function renderTable(Table $table): array
7179
return $lines;
7280
}
7381

82+
$rows = array_map($this->renderTableRow(...), $table->rows());
83+
[$columns, $rows, $isCompactTable] = $this->compactAggregateReportTable($columns, $rows);
84+
$rows = $this->filterOutlierRows($rows, $outlierMinDiff, $isCompactTable);
85+
86+
if ($rows === [] && $isCompactTable && $outlierMinDiff !== null && $outlierMinDiff > 0.0) {
87+
$lines[] = sprintf('_No benchmark changes above ±%s%%._', $this->formatPercentage($outlierMinDiff));
88+
$lines[] = '';
89+
90+
return $lines;
91+
}
92+
7493
$lines[] = $this->renderRow($columns);
7594
$lines[] = $this->renderSeparatorRow($columns);
7695

77-
foreach ($table as $row) {
78-
$lines[] = $this->renderDataRow($row);
96+
foreach ($rows as $row) {
97+
$lines[] = $this->renderRow($row);
7998
}
8099

81100
$lines[] = '';
@@ -96,11 +115,92 @@ private function renderSeparatorRow(array $columns): string
96115
));
97116
}
98117

99-
private function renderDataRow(TableRow $row): string
118+
private function renderTableRow(TableRow $row): array
119+
{
120+
return array_values(array_map($this->formatCell(...), iterator_to_array($row)));
121+
}
122+
123+
private function compactAggregateReportTable(array $columns, array $rows): array
100124
{
101-
$cells = array_map($this->formatCell(...), iterator_to_array($row));
125+
$columnIndexes = $this->resolveCompactSourceColumnIndexes($columns);
126+
127+
if ($columnIndexes === null) {
128+
return [$columns, $rows, false];
129+
}
130+
131+
$rows = array_map(function (array $row) use ($columnIndexes): array {
132+
$set = trim((string) $row[$columnIndexes['set']]);
133+
134+
return [
135+
sprintf('%s(%s)', $row[$columnIndexes['benchmark']], $row[$columnIndexes['subject']]),
136+
$set === '' ? '-' : $set,
137+
$row[$columnIndexes['mem_peak']],
138+
$row[$columnIndexes['mode']],
139+
$row[$columnIndexes['rstdev']],
140+
];
141+
}, $rows);
142+
143+
return [self::COMPACT_HEADERS, $rows, true];
144+
}
145+
146+
private function filterOutlierRows(array $rows, ?float $outlierMinDiff, bool $isCompactTable): array
147+
{
148+
if (! $isCompactTable || $outlierMinDiff === null || $outlierMinDiff <= 0.0) {
149+
return $rows;
150+
}
151+
152+
return array_values(array_filter($rows, function (array $row) use ($outlierMinDiff): bool {
153+
$diff = $this->extractTrailingPercentage($row[self::COMPACT_TIME_COLUMN_INDEX]);
154+
155+
if ($diff === null) {
156+
return true;
157+
}
158+
159+
return abs($diff) >= $outlierMinDiff;
160+
}));
161+
}
162+
163+
private function extractTrailingPercentage(string $cell): ?float
164+
{
165+
if (preg_match('/([+-]?\d+(?:\.\d+)?)%\s*$/', $cell, $matches) !== 1) {
166+
return null;
167+
}
168+
169+
return (float) $matches[1];
170+
}
171+
172+
private function resolveOutlierMinDiff(Config $config): ?float
173+
{
174+
if (! $config->offsetExists('outlier_min_diff')) {
175+
return null;
176+
}
177+
178+
$value = $config['outlier_min_diff'];
179+
180+
return $value === null ? null : (float) $value;
181+
}
182+
183+
private function formatPercentage(float $value): string
184+
{
185+
return rtrim(rtrim(sprintf('%.2f', $value), '0'), '.');
186+
}
187+
188+
private function resolveCompactSourceColumnIndexes(array $columns): ?array
189+
{
190+
$columnIndexes = array_flip($columns);
191+
192+
if (array_any(self::COMPACT_SOURCE_COLUMNS, fn ($column) => ! array_key_exists($column, $columnIndexes))) {
193+
return null;
194+
}
102195

103-
return $this->renderRow($cells);
196+
return [
197+
'benchmark' => $columnIndexes['benchmark'],
198+
'subject' => $columnIndexes['subject'],
199+
'set' => $columnIndexes['set'],
200+
'mem_peak' => $columnIndexes['mem_peak'],
201+
'mode' => $columnIndexes['mode'],
202+
'rstdev' => $columnIndexes['rstdev'],
203+
];
104204
}
105205

106206
private function formatCell(Node $node): string

0 commit comments

Comments
 (0)