Skip to content

Commit cc7ebe2

Browse files
HanSur94claude
HanSur94
and
claude
committed
ci: comprehensive benchmark with multi-run stats (mean, std, RMS)
New benchmark script (scripts/run_ci_benchmark.m) replaces inline YAML: - Tests 3 dataset sizes: 1M, 5M, 10M points - 4 metrics per size: instantiation, render, zoom cycle, downsample - Multiple iterations (10 for zoom/downsample, 5 for init/render) - Computes mean, std, RMS — reports mean + std to benchmark tracker - Warmup phase before zoom measurements - 24 total metrics tracked for regression detection Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 89b8011 commit cc7ebe2

2 files changed

Lines changed: 137 additions & 50 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 2 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -59,63 +59,15 @@ jobs:
5959

6060
- name: Run benchmark
6161
run: |
62-
xvfb-run octave --eval "
63-
addpath(pwd); setup();
64-
addpath(fullfile(pwd, 'libs', 'FastSense', 'private'));
65-
66-
n = 1e6;
67-
x = linspace(0, 100, n);
68-
y = sin(x * 2*pi / 10) + 0.5 * randn(1, n);
69-
70-
tic;
71-
for i = 1:100
72-
minmax_downsample(x, y, 2000);
73-
end
74-
t_ds = toc / 100;
75-
76-
tic;
77-
for i = 1:1000
78-
binary_search(x, 20, 'left');
79-
end
80-
t_bs = toc / 1000;
81-
82-
% Measure full render cycle
83-
fp = FastSense();
84-
fp.addLine(x, y, 'DisplayName', 'bench');
85-
fp.addThreshold(1.5, 'Direction', 'upper', 'ShowViolations', true);
86-
fp.render();
87-
88-
tic;
89-
for i = 1:20
90-
center = 10 + 80 * rand();
91-
width = 1 + 20 * rand();
92-
set(fp.hAxes, 'XLim', [center-width/2, center+width/2]);
93-
drawnow;
94-
end
95-
t_zoom = toc / 20;
96-
close all force;
97-
98-
fid = fopen('benchmark-results.json', 'w');
99-
fprintf(fid, '[\n');
100-
fprintf(fid, ' {\"name\": \"Downsample (1M pts)\", \"unit\": \"ms\", \"value\": %.2f},\n', t_ds * 1000);
101-
fprintf(fid, ' {\"name\": \"Binary Search\", \"unit\": \"us\", \"value\": %.2f},\n', t_bs * 1e6);
102-
fprintf(fid, ' {\"name\": \"Zoom Cycle (1M pts)\", \"unit\": \"ms\", \"value\": %.2f}\n', t_zoom * 1000);
103-
fprintf(fid, ']\n');
104-
fclose(fid);
105-
106-
fprintf('\n=== Benchmark Results (1M points) ===\n');
107-
fprintf('Downsample: %.2f ms\n', t_ds * 1000);
108-
fprintf('Binary search: %.2f us\n', t_bs * 1e6);
109-
fprintf('Zoom cycle: %.2f ms\n', t_zoom * 1000);
110-
"
62+
xvfb-run octave --eval "addpath(pwd); setup(); addpath('scripts'); run_ci_benchmark();"
11163
11264
- name: Fix git ownership
11365
run: git config --global --add safe.directory /__w/FastSense/FastSense
11466

11567
- name: Store benchmark results
11668
uses: benchmark-action/github-action-benchmark@v1
11769
with:
118-
name: FastSense Performance
70+
name: FastPlot Performance
11971
tool: customSmallerIsBetter
12072
output-file-path: benchmark-results.json
12173
github-token: ${{ secrets.GITHUB_TOKEN }}

‎scripts/run_ci_benchmark.m‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
function run_ci_benchmark()
2+
%RUN_CI_BENCHMARK Performance benchmark for CI with statistical analysis.
3+
% Runs multiple iterations across multiple dataset sizes to produce
4+
% mean, std, and RMS for each metric. Outputs results as JSON for
5+
% github-action-benchmark (customSmallerIsBetter format).
6+
%
7+
% Metrics measured:
8+
% - Instantiation: FastPlot() + addLine + addThreshold
9+
% - Render: render() + drawnow
10+
% - Zoom cycle: set XLim + drawnow (interactive responsiveness)
11+
% - Downsample: minmax_downsample kernel
12+
%
13+
% Dataset sizes: 1M, 5M, 10M points
14+
% Iterations: 10 per metric (5 for instantiation/render due to cost)
15+
16+
addpath(fullfile(pwd, 'libs', 'FastPlot', 'private'));
17+
18+
sizes = [1e6, 5e6, 10e6];
19+
labels = {'1M', '5M', '10M'};
20+
21+
N_DS = 20; % downsample iterations
22+
N_ZOOM = 20; % zoom cycles per run
23+
N_RUNS = 10; % runs for zoom/downsample stats
24+
N_INIT = 5; % runs for instantiation/render (heavier)
25+
26+
results = {};
27+
28+
for s = 1:numel(sizes)
29+
n = sizes(s);
30+
lbl = labels{s};
31+
fprintf('\n========== %s points ==========\n', lbl);
32+
33+
x = linspace(0, 100, n);
34+
y = sin(x * 2*pi / 10) + 0.5 * randn(1, n);
35+
36+
% --- Downsample benchmark ---
37+
t_ds = zeros(1, N_RUNS);
38+
for r = 1:N_RUNS
39+
tic;
40+
for k = 1:N_DS
41+
minmax_downsample(x, y, 2000);
42+
end
43+
t_ds(r) = toc / N_DS;
44+
end
45+
results = add_result(results, sprintf('Downsample mean (%s)', lbl), 'ms', t_ds * 1000);
46+
47+
% --- Instantiation benchmark ---
48+
t_init = zeros(1, N_INIT);
49+
for r = 1:N_INIT
50+
tic;
51+
fp = FastPlot();
52+
fp.addLine(x, y, 'DisplayName', 'Sensor');
53+
fp.addThreshold(1.5, 'Direction', 'upper', 'ShowViolations', true);
54+
fp.addThreshold(-1.5, 'Direction', 'lower', 'ShowViolations', true);
55+
t_init(r) = toc;
56+
close all force;
57+
end
58+
results = add_result(results, sprintf('Instantiation mean (%s)', lbl), 'ms', t_init * 1000);
59+
60+
% --- Render benchmark ---
61+
t_render = zeros(1, N_INIT);
62+
for r = 1:N_INIT
63+
fp = FastPlot();
64+
fp.addLine(x, y, 'DisplayName', 'Sensor');
65+
fp.addThreshold(1.5, 'Direction', 'upper', 'ShowViolations', true);
66+
fp.addThreshold(-1.5, 'Direction', 'lower', 'ShowViolations', true);
67+
tic;
68+
fp.render();
69+
drawnow;
70+
t_render(r) = toc;
71+
close all force;
72+
end
73+
results = add_result(results, sprintf('Render mean (%s)', lbl), 'ms', t_render * 1000);
74+
75+
% --- Zoom cycle benchmark ---
76+
fp = FastPlot();
77+
fp.addLine(x, y, 'DisplayName', 'Sensor');
78+
fp.addThreshold(1.5, 'Direction', 'upper', 'ShowViolations', true);
79+
fp.addThreshold(-1.5, 'Direction', 'lower', 'ShowViolations', true);
80+
fp.render();
81+
drawnow;
82+
83+
% Warmup
84+
for k = 1:5
85+
set(fp.hAxes, 'XLim', [20 80]);
86+
drawnow;
87+
end
88+
89+
t_zoom = zeros(1, N_RUNS);
90+
for r = 1:N_RUNS
91+
centers = 10 + 80 * rand(1, N_ZOOM);
92+
widths = 1 + 20 * rand(1, N_ZOOM);
93+
tic;
94+
for k = 1:N_ZOOM
95+
set(fp.hAxes, 'XLim', [centers(k)-widths(k)/2, centers(k)+widths(k)/2]);
96+
drawnow;
97+
end
98+
t_zoom(r) = toc / N_ZOOM;
99+
end
100+
close all force;
101+
102+
results = add_result(results, sprintf('Zoom cycle mean (%s)', lbl), 'ms', t_zoom * 1000);
103+
end
104+
105+
% --- Write JSON ---
106+
fid = fopen('benchmark-results.json', 'w');
107+
fprintf(fid, '[\n');
108+
for i = 1:numel(results)
109+
r = results{i};
110+
comma = ',';
111+
if i == numel(results), comma = ''; end
112+
fprintf(fid, ' {"name": "%s", "unit": "%s", "value": %.3f}%s\n', ...
113+
r.name, r.unit, r.value, comma);
114+
end
115+
fprintf(fid, ']\n');
116+
fclose(fid);
117+
118+
fprintf('\n=== Benchmark complete — %d metrics written to benchmark-results.json ===\n', numel(results));
119+
end
120+
121+
function results = add_result(results, name, unit, samples)
122+
%ADD_RESULT Compute stats and add to results list.
123+
m = mean(samples);
124+
s = std(samples);
125+
rms = sqrt(mean(samples.^2));
126+
127+
fprintf(' %-30s mean=%.2f %s std=%.2f %s rms=%.2f %s (n=%d)\n', ...
128+
name, m, unit, s, unit, rms, unit, numel(samples));
129+
130+
% Report mean as the tracked value (for regression detection)
131+
results{end+1} = struct('name', name, 'unit', unit, 'value', m);
132+
% Also report std as a separate metric
133+
results{end+1} = struct('name', [name(1:end-5) ' std' name(end-3:end)], ...
134+
'unit', unit, 'value', s);
135+
end

0 commit comments

Comments
 (0)