diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000000..67a681004c --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,106 @@ +name: CodSpeed +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + merge_group: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + bench-matrix: + name: Determine bench matrix + runs-on: ubuntu-24.04 + outputs: + benches: ${{ steps.benches.outputs.benches }} + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - id: benches + run: | + # Create a GitHub matrix where each entry is a bench with its + # associated crate and benchmarking mode taken from the Cargo + # metadata. + { + echo -n "benches=[" + first=true + while IFS=: read -r crate manifest_path; do + dir=$(dirname "$manifest_path") + if [ -e "$dir/benches" ]; then + for bench in "$dir/benches"/*.rs; do + bench=$(basename -s .rs "$bench") + # Set measurement mode from Cargo.toml metadata, default to "instrumentation". + # The following jq pipeline: + # - Selects the package with name == $crate + # - Looks up .metadata.bench[$bench].codspeed.mode for the benchmark + # - If not set, defaults to "instrumentation" + mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') + # FIXME: Always use "instrumentation" mode to eliminate CI VM effects. + mode="instrumentation" + if [ "$first" = true ]; then + first=false + else + echo -n "," + fi + echo -n "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" + done + fi + done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') + echo -n "]" + } >> "${GITHUB_OUTPUT}" + + benchmarks: + name: Run bench + runs-on: ubuntu-24.04 + strategy: + matrix: + bench: ${{ fromJson(needs.bench-matrix.outputs.benches) }} + needs: bench-matrix + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - uses: ./.github/actions/rust + with: + version: stable + tools: cargo-codspeed + token: ${{ secrets.GITHUB_TOKEN }} + + - id: nss-version + run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT" + + - uses: ./.github/actions/nss + with: + minimum-version: ${{ steps.nss-version.outputs.minimum }} + + - name: Build bench + env: + BENCH: ${{ matrix.bench.bench }} + CRATE: ${{ matrix.bench.crate }} + MODE: ${{ matrix.bench.mode }} + run: cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "$MODE" + + - name: Run bench & upload results + uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 + with: + mode: ${{ matrix.bench.mode }} + run: cargo codspeed run + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 31b7aa2571..79bc128c5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,21 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.4.1" @@ -50,7 +65,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "regex", @@ -191,35 +206,83 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] -name = "criterion" -version = "0.7.0" +name = "codspeed" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b847e05a34be5c38f3f2a5052178a3bd32e6b5702f3ea775efde95c483a539" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.2.16", + "glob", + "libc", + "nix", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a0e2a53beb18dec493ec133f226e0d35e8bb2fdc638ccf7351696eabee416c" +dependencies = [ + "clap", + "codspeed", + "codspeed-criterion-compat-walltime", + "colored", + "futures", + "regex", + "tokio", +] + +[[package]] +name = "codspeed-criterion-compat-walltime" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +checksum = "1f652d6e6d40bba0f5c244744db94d92a26b7f083df18692df88fb0772f1c793" dependencies = [ "anes", "cast", "ciborium", "clap", + "codspeed", "criterion-plot", - "itertools 0.13.0", + "futures", + "is-terminal", + "itertools", "num-traits", + "once_cell", "oorandom", "regex", "serde", + "serde_derive", "serde_json", "tinytemplate", "tokio", "walkdir", ] +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.52.0", +] + [[package]] name = "criterion-plot" -version = "0.6.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.13.0", + "itertools", ] [[package]] @@ -423,6 +486,17 @@ dependencies = [ "test-fixture", ] +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.3.3" @@ -437,9 +511,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "half" @@ -469,6 +543,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -631,19 +711,21 @@ dependencies = [ ] [[package]] -name = "itertools" -version = "0.10.5" +name = "is-terminal" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ - "either", + "hermit-abi 0.5.2", + "libc", + "windows-sys 0.52.0", ] [[package]] name = "itertools" -version = "0.13.0" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] @@ -660,10 +742,16 @@ version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ - "getrandom", + "getrandom 0.3.3", "libc", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.176" @@ -720,7 +808,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", @@ -750,7 +838,7 @@ version = "0.18.0" dependencies = [ "clap", "clap-verbosity-flag", - "criterion", + "codspeed-criterion-compat", "futures", "hex", "log", @@ -773,7 +861,7 @@ dependencies = [ name = "neqo-common" version = "0.18.0" dependencies = [ - "criterion", + "codspeed-criterion-compat", "enum-map", "env_logger", "hex", @@ -810,7 +898,7 @@ dependencies = [ name = "neqo-http3" version = "0.18.0" dependencies = [ - "criterion", + "codspeed-criterion-compat", "enumset", "log", "neqo-common", @@ -845,7 +933,7 @@ dependencies = [ name = "neqo-transport" version = "0.18.0" dependencies = [ - "criterion", + "codspeed-criterion-compat", "enum-map", "enumset", "indexmap", @@ -875,6 +963,18 @@ dependencies = [ "windows", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1180,6 +1280,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strsim" version = "0.11.1" diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index d5001081fb..a28399cf34 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -52,9 +52,8 @@ tokio = { version = "1", default-features = false, features = ["net", "time", "m url = { workspace = true } [dev-dependencies] -criterion = { version = "0.7", default-features = false, features = [ +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false, features = [ "async_tokio", - "cargo_bench_support", ] } neqo-bin = { path = ".", features = ["draft-29"] } tokio = { version = "1", default-features = false, features = ["sync"] } @@ -71,6 +70,9 @@ ignored = ["log"] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options bench = false +[package.metadata.bench.main] +codspeed.mode = "walltime" # Use "walltime" mode for "main" bench with codspeed. + [[bench]] name = "main" harness = false diff --git a/neqo-bin/benches/main.rs b/neqo-bin/benches/main.rs index f8ddaa75dc..163d6d1af3 100644 --- a/neqo-bin/benches/main.rs +++ b/neqo-bin/benches/main.rs @@ -5,6 +5,10 @@ // except according to those terms. #![expect(clippy::unwrap_used, reason = "OK in a bench.")] +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] use std::{env, hint::black_box, net::SocketAddr, path::PathBuf, str::FromStr as _}; diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml index 0d838e75d4..33c064ef53 100644 --- a/neqo-common/Cargo.toml +++ b/neqo-common/Cargo.toml @@ -28,7 +28,7 @@ thiserror = { workspace = true } windows = { workspace = true, features = ["Win32_Media"] } [dev-dependencies] -criterion = { version = "0.7", default-features = false, features = ["cargo_bench_support"] } +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false } neqo-crypto = { path = "../neqo-crypto" } test-fixture = { path = "../test-fixture" } regex = { workspace = true } diff --git a/neqo-common/benches/decoder.rs b/neqo-common/benches/decoder.rs index d3edc3e2d2..2ec8935c9f 100644 --- a/neqo-common/benches/decoder.rs +++ b/neqo-common/benches/decoder.rs @@ -5,6 +5,10 @@ // except according to those terms. #![expect(clippy::unwrap_used, reason = "OK in a bench.")] +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] use std::hint::black_box; diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index 5fa887f999..77d64d7623 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -33,7 +33,7 @@ thiserror = { workspace = true } url = { workspace = true } [dev-dependencies] -criterion = { version = "0.7", default-features = false, features = ["cargo_bench_support"] } +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false } neqo-http3 = { path = ".", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } @@ -55,6 +55,9 @@ ignored = ["criterion", "log"] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options bench = false +[package.metadata.bench.streams] +codspeed.mode = "walltime" # Use "walltime" mode for "streams" bench with codspeed. + [[bench]] name = "streams" harness = false diff --git a/neqo-http3/benches/streams.rs b/neqo-http3/benches/streams.rs index b77dabb498..c9ff4c6ac3 100644 --- a/neqo-http3/benches/streams.rs +++ b/neqo-http3/benches/streams.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::{hint::black_box, time::Duration}; use criterion::{criterion_group, criterion_main, Criterion, Throughput}; @@ -62,8 +67,6 @@ fn criterion_benchmark(c: &mut Criterion) { group.finish(); } - - Criterion::default().configure_from_args().final_summary(); } criterion_group!(benches, criterion_benchmark); diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml index b9baff581a..4be52ab21d 100644 --- a/neqo-transport/Cargo.toml +++ b/neqo-transport/Cargo.toml @@ -32,7 +32,7 @@ strum = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -criterion = { version = "0.7", default-features = false, features = ["cargo_bench_support"] } +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false } neqo-transport = { path = ".", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } diff --git a/neqo-transport/benches/range_tracker.rs b/neqo-transport/benches/range_tracker.rs index 02094f1fa4..93bb2b3abb 100644 --- a/neqo-transport/benches/range_tracker.rs +++ b/neqo-transport/benches/range_tracker.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::hint::black_box; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/neqo-transport/benches/rx_stream_orderer.rs b/neqo-transport/benches/rx_stream_orderer.rs index c4a73fb85d..8e01e283f8 100644 --- a/neqo-transport/benches/rx_stream_orderer.rs +++ b/neqo-transport/benches/rx_stream_orderer.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::hint::black_box; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/neqo-transport/benches/sent_packets.rs b/neqo-transport/benches/sent_packets.rs index d6c812f893..8315f136d8 100644 --- a/neqo-transport/benches/sent_packets.rs +++ b/neqo-transport/benches/sent_packets.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::{hint::black_box, time::Instant}; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/neqo-transport/benches/transfer.rs b/neqo-transport/benches/transfer.rs index 1fbc2c1b30..2c48a90988 100644 --- a/neqo-transport/benches/transfer.rs +++ b/neqo-transport/benches/transfer.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::{hint::black_box, time::Duration}; use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion};