Skip to content

Commit 5c17a43

Browse files
Akagi201Copilot
andcommitted
Add benchmark reports and scripts for performance analysis
- Introduced max_perf.json and core_interfaces.json to store benchmark results for maximum performance and core interfaces. - Added runtime-baselines for max_perf and simple_bench to track performance metrics. - Created criterion_compare.py to compare benchmark reports and generate summaries. - Implemented criterion_export.py to export benchmark estimates into structured reports. - Developed hotpath_compare.py to analyze hotpath benchmark reports and summarize performance metrics. Co-authored-by: Copilot <copilot@github.com>
1 parent 5769992 commit 5c17a43

28 files changed

Lines changed: 2500 additions & 246 deletions

.github/workflows/ci.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,18 @@ jobs:
5050

5151
- name: CI
5252
run: just ci
53+
54+
- name: Run hotpath benchmark baselines
55+
run: just bench-hotpath
56+
57+
- name: Compare hotpath output against baseline
58+
run: just bench-baseline-compare
59+
60+
- name: Upload hotpath benchmark artifacts
61+
if: always()
62+
uses: actions/upload-artifact@v4
63+
with:
64+
name: hotpath-benchmarks
65+
path: |
66+
target/hotpath/*.json
67+
target/hotpath/summary.md

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ cucumber = "0.23.0"
2525
ed25519-dalek = "=1.0.1"
2626
hashbrown = "0.17.0"
2727
hex = "0.4.3"
28+
hotpath = "0.15.1"
2829
indexmap = "2.14.0"
2930
itertools = "0.14.0"
3031
libsecp256k1 = "0.7.2"

Justfile

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,45 @@ test-coverage:
4343
build:
4444
cargo build --workspace
4545

46+
# Run hotpath-enabled benchmark baselines
47+
bench-hotpath:
48+
mkdir -p target/hotpath
49+
HPSVM_HOTPATH=1 cargo bench -p hpsvm --features hotpath --bench core_interfaces -- --noplot --sample-size 10 --measurement-time 1 --warm-up-time 1
50+
HPSVM_HOTPATH=1 cargo bench -p hpsvm --features hotpath --bench max_perf -- --noplot --sample-size 10 --measurement-time 1 --warm-up-time 1
51+
52+
# Run the steady-state benchmark with hotpath and register trace metrics enabled
53+
bench-hotpath-trace:
54+
mkdir -p target/hotpath
55+
HPSVM_HOTPATH=1 HPSVM_TRACE_METRICS=1 cargo bench -p hpsvm --features "hotpath register-tracing" --bench max_perf -- --noplot --sample-size 10 --measurement-time 1 --warm-up-time 1
56+
HPSVM_HOTPATH=1 HPSVM_TRACE_METRICS=1 cargo bench -p hpsvm --features "hotpath register-tracing" --bench simple_bench -- --noplot --sample-size 10 --measurement-time 1 --warm-up-time 1
57+
58+
# Run default runtime benchmarks without hotpath overhead
59+
bench-runtime:
60+
cargo bench -p hpsvm --bench simple_bench -- --noplot --sample-size 10 --measurement-time 1 --warm-up-time 1
61+
cargo bench -p hpsvm --bench max_perf -- --noplot --sample-size 10 --measurement-time 1 --warm-up-time 1
62+
cargo bench -p hpsvm --bench core_interfaces -- --noplot --sample-size 10 --measurement-time 1 --warm-up-time 1
63+
64+
# Refresh committed default runtime baselines from the current machine
65+
bench-runtime-baseline-refresh: bench-runtime
66+
mkdir -p docs/benchmarks/runtime-baselines
67+
python3 scripts/criterion_export.py target/criterion docs/benchmarks/runtime-baselines --include simple_bench --include max_perf --include core_interfaces
68+
69+
# Compare a fresh default runtime benchmark run against committed baselines
70+
bench-runtime-baseline-compare:
71+
mkdir -p target/runtime-benchmarks
72+
python3 scripts/criterion_export.py target/criterion target/runtime-benchmarks --include simple_bench --include max_perf --include core_interfaces
73+
python3 scripts/criterion_compare.py docs/benchmarks/runtime-baselines target/runtime-benchmarks --output target/runtime-benchmarks/summary.md
74+
75+
# Refresh committed hotpath benchmark baselines from the current machine
76+
bench-baseline-refresh: bench-hotpath
77+
mkdir -p docs/benchmarks/baselines
78+
cp target/hotpath/core_interfaces.json docs/benchmarks/baselines/core_interfaces.json
79+
cp target/hotpath/max_perf.json docs/benchmarks/baselines/max_perf.json
80+
81+
# Compare current hotpath benchmark output against committed baselines
82+
bench-baseline-compare:
83+
python3 scripts/hotpath_compare.py docs/benchmarks/baselines target/hotpath --output target/hotpath/summary.md
84+
4685
# Check all targets compile
4786
check:
4887
cargo check --all-targets --all-features

README.md

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,70 @@ cargo test
204204
cargo bench
205205
```
206206

207+
Build the benchmark helper program and profile the public HPSVM surface:
208+
209+
```bash
210+
HPSVM_HOTPATH=1 cargo bench -p hpsvm --features hotpath --bench core_interfaces
211+
```
212+
213+
Profile the steady-state transaction execution hot path:
214+
215+
```bash
216+
HPSVM_HOTPATH=1 cargo bench -p hpsvm --features hotpath --bench max_perf
217+
```
218+
219+
Export a deeper trace summary for the steady-state SBF execution path:
220+
221+
```bash
222+
just bench-hotpath-trace
223+
```
224+
225+
`core_interfaces` covers the main public APIs: `new`, `set_account`, `get_account`,
226+
`expire_blockhash`, `add_program`, `add_program_from_file`, `airdrop`,
227+
`send_transaction`, `simulate_transaction`, `transact` + `commit_transaction`,
228+
`plan_transaction_batch`, and `send_transaction_batch`.
229+
230+
When `HPSVM_HOTPATH=1` is set, hotpath reports are written to `target/hotpath/*.json`.
231+
Use `HPSVM_HOTPATH_LIMIT` to cap the number of reported functions, or `HOTPATH_OUTPUT_PATH`
232+
to override the default report path.
233+
234+
When `HPSVM_TRACE_METRICS=1` is combined with the `register-tracing` feature, `just bench-hotpath-trace`
235+
also writes `target/hotpath/max_perf.trace.json` and `target/hotpath/simple_bench.trace.json`.
236+
These summaries report per-program invocation counts, CPI depth, register-frame counts, and
237+
instruction-account counts so you can tell whether the `execute_instruction` hotspot is dominated
238+
by nested CPIs or a single SBF program body in both the steady-state loop and the repeated
239+
transaction-loop benchmark.
240+
241+
To compare default runtime benchmarks without hotpath overhead:
242+
243+
```bash
244+
just bench-runtime
245+
just bench-runtime-baseline-compare
246+
```
247+
248+
This writes compact Criterion reports to `target/runtime-benchmarks/*.json` and a markdown summary
249+
to `target/runtime-benchmarks/summary.md`. Use this runtime baseline before treating a hotpath-only
250+
delta as a real VM slowdown.
251+
252+
To refresh the committed default runtime baselines:
253+
254+
```bash
255+
just bench-runtime-baseline-refresh
256+
```
257+
258+
To refresh the committed hotpath baselines:
259+
260+
```bash
261+
just bench-baseline-refresh
262+
```
263+
264+
To compare a fresh benchmark run with the committed baselines:
265+
266+
```bash
267+
just bench-hotpath
268+
just bench-baseline-compare
269+
```
270+
207271
### Code Quality
208272

209273
Format code:

crates/hpsvm/Cargo.toml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ hashbrown = ["dep:hashbrown"]
2020
serde = []
2121
precompiles = ["dep:agave-precompiles"]
2222
register-tracing = ["invocation-inspect-callback", "dep:hex", "dep:sha2"]
23+
hotpath = ["dep:hotpath"]
2324

2425
[dependencies]
2526
agave-feature-set.workspace = true
@@ -33,6 +34,7 @@ agave-syscalls.workspace = true
3334
ansi_term.workspace = true
3435
hashbrown = { workspace = true, optional = true }
3536
hex = { workspace = true, optional = true }
37+
hotpath = { workspace = true, optional = true, features = ["hotpath"] }
3638
indexmap.workspace = true
3739
itertools.workspace = true
3840
log.workspace = true
@@ -131,3 +133,7 @@ harness = false
131133
[[bench]]
132134
name = "hashing"
133135
harness = false
136+
137+
[[bench]]
138+
name = "core_interfaces"
139+
harness = false

crates/hpsvm/benches/common/mod.rs

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
#![allow(dead_code)]
2+
3+
use std::{env, path::PathBuf};
4+
5+
use hpsvm::HPSVM;
6+
#[cfg(feature = "register-tracing")]
7+
use hpsvm::register_tracing::TraceMetricsCollector;
8+
use solana_account::Account;
9+
use solana_address::Address;
10+
use solana_instruction::{Instruction, account_meta::AccountMeta};
11+
use solana_keypair::Keypair;
12+
use solana_message::Message;
13+
use solana_transaction::Transaction;
14+
15+
const COUNTER_PROGRAM_RELATIVE_PATH: &str = "test_programs/target/deploy/counter.so";
16+
const HOTPATH_ENV_VAR: &str = "HPSVM_HOTPATH";
17+
const HOTPATH_LIMIT_ENV_VAR: &str = "HPSVM_HOTPATH_LIMIT";
18+
const TRACE_METRICS_ENV_VAR: &str = "HPSVM_TRACE_METRICS";
19+
20+
pub struct HotpathGuard {
21+
#[cfg(feature = "hotpath")]
22+
_inner: Option<hotpath::HotpathGuard>,
23+
}
24+
25+
impl HotpathGuard {
26+
pub fn new(name: &'static str) -> Self {
27+
#[cfg(feature = "hotpath")]
28+
{
29+
if env::var_os(HOTPATH_ENV_VAR).is_none() {
30+
return Self { _inner: None };
31+
}
32+
33+
let limit = env::var(HOTPATH_LIMIT_ENV_VAR)
34+
.ok()
35+
.and_then(|value| value.parse::<usize>().ok())
36+
.unwrap_or(20);
37+
let output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
38+
.join("../../target/hotpath")
39+
.join(format!("{name}.json"));
40+
if let Some(parent) = output_path.parent() {
41+
std::fs::create_dir_all(parent)
42+
.expect("hotpath benchmark output directory should be creatable");
43+
}
44+
let guard = hotpath::HotpathGuardBuilder::new(name)
45+
.percentiles(&[50.0, 95.0, 99.0])
46+
.format(hotpath::Format::JsonPretty)
47+
.output_path(&output_path)
48+
.limit(limit)
49+
.build();
50+
51+
return Self { _inner: Some(guard) };
52+
}
53+
54+
#[cfg(not(feature = "hotpath"))]
55+
{
56+
let _ = name;
57+
Self {}
58+
}
59+
}
60+
}
61+
62+
pub struct TraceMetricsGuard {
63+
#[cfg(feature = "register-tracing")]
64+
collector: Option<TraceMetricsCollector>,
65+
#[cfg(feature = "register-tracing")]
66+
output_path: Option<PathBuf>,
67+
}
68+
69+
impl TraceMetricsGuard {
70+
pub fn new(name: &'static str) -> Self {
71+
#[cfg(feature = "register-tracing")]
72+
{
73+
if env::var_os(TRACE_METRICS_ENV_VAR).is_none() {
74+
return Self { collector: None, output_path: None };
75+
}
76+
77+
let output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
78+
.join("../../target/hotpath")
79+
.join(format!("{name}.trace.json"));
80+
if let Some(parent) = output_path.parent() {
81+
std::fs::create_dir_all(parent)
82+
.expect("trace metrics output directory should be creatable");
83+
}
84+
85+
return Self {
86+
collector: Some(TraceMetricsCollector::default()),
87+
output_path: Some(output_path),
88+
};
89+
}
90+
91+
#[cfg(not(feature = "register-tracing"))]
92+
{
93+
let _ = name;
94+
Self {}
95+
}
96+
}
97+
98+
pub fn install(&self, svm: &mut HPSVM) {
99+
#[cfg(feature = "register-tracing")]
100+
if let Some(collector) = &self.collector {
101+
svm.set_invocation_inspect_callback(collector.clone());
102+
}
103+
104+
#[cfg(not(feature = "register-tracing"))]
105+
let _ = svm;
106+
}
107+
}
108+
109+
impl Drop for TraceMetricsGuard {
110+
fn drop(&mut self) {
111+
#[cfg(feature = "register-tracing")]
112+
if let (Some(collector), Some(output_path)) = (&self.collector, &self.output_path) {
113+
collector
114+
.write_json_path(output_path)
115+
.expect("trace metrics output should be writable");
116+
}
117+
}
118+
}
119+
120+
pub fn new_benchmark_vm() -> HPSVM {
121+
#[cfg(feature = "register-tracing")]
122+
let mut svm = if env::var_os(TRACE_METRICS_ENV_VAR).is_some() {
123+
HPSVM::new_debuggable(true)
124+
} else {
125+
HPSVM::new()
126+
};
127+
128+
#[cfg(not(feature = "register-tracing"))]
129+
let mut svm = HPSVM::new();
130+
131+
svm.set_blockhash_check(false);
132+
svm.set_sigverify(false);
133+
svm.set_transaction_history(0);
134+
svm
135+
}
136+
137+
pub fn counter_program_path() -> PathBuf {
138+
let mut so_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
139+
so_path.push(COUNTER_PROGRAM_RELATIVE_PATH);
140+
so_path
141+
}
142+
143+
pub fn read_counter_program() -> Vec<u8> {
144+
std::fs::read(counter_program_path()).expect("counter program bytes should be available")
145+
}
146+
147+
pub fn make_counter_tx(
148+
program_id: Address,
149+
counter_address: Address,
150+
payer_pk: &Address,
151+
blockhash: solana_hash::Hash,
152+
payer_kp: &Keypair,
153+
deduper: u8,
154+
) -> Transaction {
155+
let msg = Message::new_with_blockhash(
156+
&[Instruction {
157+
program_id,
158+
accounts: vec![AccountMeta::new(counter_address, false)],
159+
data: vec![0, deduper],
160+
}],
161+
Some(payer_pk),
162+
&blockhash,
163+
);
164+
Transaction::new(&[payer_kp], msg, blockhash)
165+
}
166+
167+
pub fn counter_account(program_id: Address) -> Account {
168+
Account {
169+
lamports: 5,
170+
data: vec![0_u8; std::mem::size_of::<u32>()],
171+
owner: program_id,
172+
..Default::default()
173+
}
174+
}

0 commit comments

Comments
 (0)