Skip to content

Commit 4a43be7

Browse files
vitormattosvitormattos
committed
perf(ci): add composer baseline update and stale-check scripts
Signed-off-by: Vitor Mattos <1079143+vitormattos@users.noreply.github.com>
1 parent 3d064dc commit 4a43be7

5 files changed

Lines changed: 186 additions & 61 deletions

File tree

‎.github/.performance/baseline.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"version": "1.0.0",
33
"created_at": "2026-06-01",
44
"allowed_regression_pct": 25.0,
5+
"stale_threshold_pct": 35.0,
56
"benchmarks": {
67
"LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchSimpleHtml": {
78
"mean": 0.356085,

‎.github/copilot-instructions.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,15 @@ composer benchmark:run
2020
# Run with stricter settings (multiple revisions)
2121
composer benchmark:run:ci
2222

23+
# Update baseline after accepted performance changes
24+
composer benchmark:baseline:update
25+
26+
# Check PR against baseline (regression gate)
27+
composer benchmark:baseline:check
28+
29+
# Check whether baseline is stale and should be refreshed
30+
composer benchmark:baseline:stale
31+
2332
# Direct PHPBench invocation with custom options
2433
vendor-bin/phpbench/vendor/phpbench/phpbench/bin/phpbench run --bootstrap=vendor/autoload.php --report=aggregate benchmarks
2534
```

‎.github/workflows/performance.yml‎

Lines changed: 4 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -40,68 +40,11 @@ jobs:
4040
4141
- name: Validate regression against baseline
4242
if: github.event_name == 'pull_request'
43-
run: |
44-
python3 - <<'PY'
45-
import json
46-
import pathlib
47-
import re
48-
import sys
49-
50-
baseline_path = pathlib.Path('.github/.performance/baseline.json')
51-
output_path = pathlib.Path('build/benchmark-output.txt')
52-
53-
if not baseline_path.exists():
54-
print('Baseline file not found, skipping regression validation.')
55-
sys.exit(0)
56-
57-
if not output_path.exists():
58-
print('Benchmark output not found, cannot validate regression.')
59-
sys.exit(1)
60-
61-
baseline = json.loads(baseline_path.read_text())
62-
output = output_path.read_text()
63-
64-
patterns = {
65-
'LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchSimpleHtml': r'benchSimpleHtml.*?Mo([0-9]+(?:\\.[0-9]+)?)(μs|us|ms)',
66-
'LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchComplexHtml': r'benchComplexHtml.*?Mo([0-9]+(?:\\.[0-9]+)?)(μs|us|ms)',
67-
}
68-
69-
tolerance = float(baseline.get('allowed_regression_pct', 25.0)) / 100.0
70-
benchmark_baseline = baseline.get('benchmarks', {})
71-
72-
failures = []
73-
for name, pattern in patterns.items():
74-
match = re.search(pattern, output)
75-
if not match:
76-
failures.append(f'{name}: metric not found in benchmark output')
77-
continue
43+
run: composer benchmark:baseline:check
7844

79-
value = float(match.group(1))
80-
unit = match.group(2)
81-
current_ms = value / 1000.0 if unit in ('μs', 'us') else value
82-
83-
if name not in benchmark_baseline:
84-
failures.append(f'{name}: missing baseline entry')
85-
continue
86-
87-
baseline_ms = float(benchmark_baseline[name]['mean'])
88-
limit_ms = baseline_ms * (1.0 + tolerance)
89-
90-
print(f'{name}: current={current_ms:.6f}ms baseline={baseline_ms:.6f}ms limit={limit_ms:.6f}ms')
91-
92-
if current_ms > limit_ms:
93-
failures.append(
94-
f'{name}: regression detected ({current_ms:.6f}ms > {limit_ms:.6f}ms)'
95-
)
96-
97-
if failures:
98-
print('\nPerformance regression gate failed:')
99-
for failure in failures:
100-
print(f' - {failure}')
101-
sys.exit(1)
102-
103-
print('\nPerformance regression gate passed.')
104-
PY
45+
- name: Validate baseline freshness
46+
if: github.event_name == 'pull_request'
47+
run: composer benchmark:baseline:stale
10548

10649
- name: Save benchmark results as artifact
10750
if: always()

‎benchmarks/baseline.php‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
<?php
2+
3+
// SPDX-FileCopyrightText: 2026 LibreSign
4+
// SPDX-License-Identifier: AGPL-3.0-or-later
5+
6+
declare(strict_types=1);
7+
8+
$root = dirname(__DIR__);
9+
$baselinePath = $root . '/.github/.performance/baseline.json';
10+
$outputPath = $root . '/build/benchmark-output.txt';
11+
12+
$command = $argv[1] ?? null;
13+
if ($command === null || !in_array($command, ['update', 'check', 'check-stale'], true)) {
14+
fwrite(STDERR, "Usage: php benchmarks/baseline.php <update|check|check-stale>\n");
15+
exit(1);
16+
}
17+
18+
if (!file_exists($outputPath)) {
19+
fwrite(STDERR, "Benchmark output file not found: {$outputPath}\n");
20+
exit(1);
21+
}
22+
23+
$output = file_get_contents($outputPath);
24+
if ($output === false) {
25+
fwrite(STDERR, "Unable to read benchmark output file.\n");
26+
exit(1);
27+
}
28+
29+
$patterns = [
30+
'LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchSimpleHtml' => '/benchSimpleHtml.*?Mo([0-9]+(?:\\.[0-9]+)?)(μs|us|ms)/',
31+
'LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchComplexHtml' => '/benchComplexHtml.*?Mo([0-9]+(?:\\.[0-9]+)?)(μs|us|ms)/',
32+
];
33+
34+
$current = [];
35+
foreach ($patterns as $name => $pattern) {
36+
if (!preg_match($pattern, $output, $matches)) {
37+
fwrite(STDERR, "Metric not found in benchmark output for {$name}.\n");
38+
exit(1);
39+
}
40+
41+
$value = (float) $matches[1];
42+
$unit = $matches[2];
43+
$currentMs = $unit === 'ms' ? $value : ($value / 1000.0);
44+
45+
$current[$name] = $currentMs;
46+
}
47+
48+
if ($command === 'update') {
49+
$baseline = [
50+
'version' => '1.0.0',
51+
'created_at' => gmdate('Y-m-d'),
52+
'allowed_regression_pct' => 25.0,
53+
'stale_threshold_pct' => 35.0,
54+
'benchmarks' => [
55+
'LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchSimpleHtml' => [
56+
'mean' => round($current['LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchSimpleHtml'], 6),
57+
'memory_real' => 768,
58+
],
59+
'LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchComplexHtml' => [
60+
'mean' => round($current['LibreSign\\XObjectTemplate\\Benchmarks\\CompilerBench::benchComplexHtml'], 6),
61+
'memory_real' => 1024,
62+
],
63+
],
64+
];
65+
66+
$encoded = json_encode($baseline, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
67+
if ($encoded === false) {
68+
fwrite(STDERR, "Unable to encode baseline JSON.\n");
69+
exit(1);
70+
}
71+
72+
$dir = dirname($baselinePath);
73+
if (!is_dir($dir) && !mkdir($dir, 0777, true) && !is_dir($dir)) {
74+
fwrite(STDERR, "Unable to create baseline directory.\n");
75+
exit(1);
76+
}
77+
78+
if (file_put_contents($baselinePath, $encoded . "\n") === false) {
79+
fwrite(STDERR, "Unable to write baseline file.\n");
80+
exit(1);
81+
}
82+
83+
fwrite(STDOUT, "Baseline updated: {$baselinePath}\n");
84+
exit(0);
85+
}
86+
87+
if (!file_exists($baselinePath)) {
88+
fwrite(STDERR, "Baseline file not found: {$baselinePath}\n");
89+
exit(1);
90+
}
91+
92+
$baselineJson = file_get_contents($baselinePath);
93+
if ($baselineJson === false) {
94+
fwrite(STDERR, "Unable to read baseline file.\n");
95+
exit(1);
96+
}
97+
98+
$baseline = json_decode($baselineJson, true);
99+
if (!is_array($baseline)) {
100+
fwrite(STDERR, "Invalid baseline JSON format.\n");
101+
exit(1);
102+
}
103+
104+
$baselineBenchmarks = $baseline['benchmarks'] ?? [];
105+
if (!is_array($baselineBenchmarks)) {
106+
fwrite(STDERR, "Invalid baseline benchmark section.\n");
107+
exit(1);
108+
}
109+
110+
$allowedRegressionPct = (float) ($baseline['allowed_regression_pct'] ?? 25.0);
111+
$staleThresholdPct = (float) ($baseline['stale_threshold_pct'] ?? 35.0);
112+
113+
$failures = [];
114+
foreach ($current as $name => $currentMs) {
115+
if (!isset($baselineBenchmarks[$name]['mean'])) {
116+
$failures[] = "{$name}: missing baseline entry";
117+
continue;
118+
}
119+
120+
$baselineMs = (float) $baselineBenchmarks[$name]['mean'];
121+
if ($baselineMs <= 0) {
122+
$failures[] = "{$name}: invalid baseline mean {$baselineMs}";
123+
continue;
124+
}
125+
126+
$deltaPct = (($currentMs - $baselineMs) / $baselineMs) * 100.0;
127+
$limitPct = $command === 'check' ? $allowedRegressionPct : $staleThresholdPct;
128+
129+
fwrite(
130+
STDOUT,
131+
sprintf(
132+
"%s: current=%.6fms baseline=%.6fms delta=%+.2f%% limit=%s%.2f%%\n",
133+
$name,
134+
$currentMs,
135+
$baselineMs,
136+
$deltaPct,
137+
$command === 'check' ? '+' : '±',
138+
$limitPct,
139+
),
140+
);
141+
142+
if ($command === 'check' && $deltaPct > $allowedRegressionPct) {
143+
$failures[] = sprintf(
144+
'%s: regression detected (%+.2f%% > +%.2f%%)',
145+
$name,
146+
$deltaPct,
147+
$allowedRegressionPct,
148+
);
149+
}
150+
151+
if ($command === 'check-stale' && abs($deltaPct) > $staleThresholdPct) {
152+
$failures[] = sprintf(
153+
'%s: baseline appears stale (%+.2f%% exceeds ±%.2f%%). Run composer benchmark:baseline:update',
154+
$name,
155+
$deltaPct,
156+
$staleThresholdPct,
157+
);
158+
}
159+
}
160+
161+
if ($failures !== []) {
162+
fwrite(STDERR, "\nBaseline validation failed:\n");
163+
foreach ($failures as $failure) {
164+
fwrite(STDERR, " - {$failure}\n");
165+
}
166+
exit(1);
167+
}
168+
169+
fwrite(STDOUT, "\nBaseline validation passed.\n");

‎composer.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@
7777
"mutation:test": "vendor-bin/mutation/vendor/infection/infection/bin/infection --threads=max",
7878
"benchmark:run": "vendor-bin/phpbench/vendor/phpbench/phpbench/bin/phpbench run --bootstrap=vendor/autoload.php --report=aggregate benchmarks",
7979
"benchmark:run:ci": "mkdir -p build && vendor-bin/phpbench/vendor/phpbench/phpbench/bin/phpbench run --bootstrap=vendor/autoload.php --iterations=20 --revs=10 --warmup=2 --report=aggregate --dump-file=build/benchmark-results.xml benchmarks",
80+
"benchmark:baseline:update": "composer benchmark:run:ci | tee build/benchmark-output.txt && php benchmarks/baseline.php update",
81+
"benchmark:baseline:check": "php benchmarks/baseline.php check",
82+
"benchmark:baseline:stale": "php benchmarks/baseline.php check-stale",
8083
"bin": [
8184
"composer bin all list"
8285
]

0 commit comments

Comments
 (0)