From de041f91bfa8618049522f84821b0d9d0aa70857 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 24 Oct 2025 13:48:30 +0300 Subject: [PATCH 01/28] ci: Use `codspeed` for benchmarking WIP --- Cargo.lock | 326 +++++++++++++++++++++++++++++++++++--- neqo-bin/Cargo.toml | 5 +- neqo-common/Cargo.toml | 2 +- neqo-http3/Cargo.toml | 2 +- neqo-transport/Cargo.toml | 2 +- 5 files changed, 305 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dd4e74557..3b4905aa11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,21 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.3.2" @@ -80,7 +95,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools", "lazy_static", "lazycell", "proc-macro2", @@ -97,6 +112,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + [[package]] name = "cast" version = "0.3.0" @@ -223,27 +244,77 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] -name = "criterion" -version = "0.6.0" +name = "codspeed" +version = "4.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679" +checksum = "fe68fdd3fe25bc26de0230718d74eb150f09f70be3141c61ea7f9b00812054aa" +dependencies = [ + "anyhow", + "cc", + "colored", + "glob", + "libc", + "nix", + "serde", + "serde_json", + "statrs", + "uuid", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "4.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a06533c3c2c7b43b2efcca2dc0f228097b4c582c95f59cc187bbeb926b42cfb" +dependencies = [ + "clap", + "codspeed", + "codspeed-criterion-compat-walltime", + "colored", + "futures", + "regex", + "tokio", +] + +[[package]] +name = "codspeed-criterion-compat-walltime" +version = "4.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4102bd4dcfb09fae5cf5ca86c6c5ad0bd0cabe834cd84b5e7b7dd969fb9093e8" dependencies = [ "anes", "cast", "ciborium", "clap", + "codspeed", "criterion-plot", - "itertools 0.13.0", + "futures", + "is-terminal", + "itertools", "num-traits", + "once_cell", "oorandom", + "plotters", + "rayon", "regex", "serde", + "serde_derive", "serde_json", "tinytemplate", "tokio", "walkdir", ] +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys", +] + [[package]] name = "criterion-plot" version = "0.5.0" @@ -251,9 +322,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.10.5", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.2" @@ -475,9 +571,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "half" @@ -507,6 +603,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -669,19 +771,21 @@ dependencies = [ ] [[package]] -name = "itertools" -version = "0.10.5" +name = "is-terminal" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ - "either", + "hermit-abi 0.5.2", + "libc", + "windows-sys", ] [[package]] name = "itertools" -version = "0.13.0" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] @@ -702,6 +806,16 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -779,7 +893,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", @@ -809,7 +923,7 @@ version = "0.17.0" dependencies = [ "clap", "clap-verbosity-flag", - "criterion", + "codspeed-criterion-compat", "futures", "hex", "log", @@ -832,7 +946,7 @@ dependencies = [ name = "neqo-common" version = "0.17.0" dependencies = [ - "criterion", + "codspeed-criterion-compat", "enum-map", "env_logger", "hex", @@ -868,7 +982,7 @@ dependencies = [ name = "neqo-http3" version = "0.17.0" dependencies = [ - "criterion", + "codspeed-criterion-compat", "enumset", "log", "neqo-common", @@ -903,7 +1017,7 @@ dependencies = [ name = "neqo-transport" version = "0.17.0" dependencies = [ - "criterion", + "codspeed-criterion-compat", "enum-map", "enumset", "indexmap", @@ -933,6 +1047,18 @@ dependencies = [ "windows", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -991,6 +1117,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -1041,6 +1195,26 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "ref-cast" version = "1.0.24" @@ -1137,18 +1311,28 @@ checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -1157,14 +1341,16 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "indexmap", "itoa", + "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -1238,6 +1424,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1404,6 +1600,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "walkdir" version = "2.3.2" @@ -1430,6 +1637,75 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index f5a227f59f..334a7335a4 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -52,10 +52,7 @@ tokio = { version = "1", default-features = false, features = ["net", "time", "m url = { workspace = true } [dev-dependencies] -criterion = { version = "0.6", default-features = false, features = [ - "async_tokio", - "cargo_bench_support", -] } +criterion = { version = "4.0", package = "codspeed-criterion-compat", features = ["async_tokio"] } neqo-bin = { path = ".", features = ["draft-29"] } neqo-http3 = { path = "./../neqo-http3", features = ["draft-29"] } neqo-transport = { path = "./../neqo-transport", features = ["draft-29"] } diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml index 43bbc3313f..d4fcedfd5b 100644 --- a/neqo-common/Cargo.toml +++ b/neqo-common/Cargo.toml @@ -28,7 +28,7 @@ thiserror = { workspace = true } windows = { workspace = true, features = ["Win32_Media"] } [dev-dependencies] -criterion = { version = "0.6", default-features = false, features = ["cargo_bench_support"] } +criterion = { version = "4.0", package = "codspeed-criterion-compat" } neqo-crypto = { path = "../neqo-crypto" } test-fixture = { path = "../test-fixture" } regex = { workspace = true } diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index b4e1e0dcfb..adea4a14ac 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -33,7 +33,7 @@ thiserror = { workspace = true } url = { workspace = true } [dev-dependencies] -criterion = { version = "0.6", default-features = false, features = ["cargo_bench_support"] } +criterion = { version = "4.0", package = "codspeed-criterion-compat" } neqo-http3 = { path = ".", features = ["draft-29"] } neqo-transport = { path = "./../neqo-transport", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml index 4e4edc08c9..8d97338be8 100644 --- a/neqo-transport/Cargo.toml +++ b/neqo-transport/Cargo.toml @@ -32,7 +32,7 @@ strum = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -criterion = { version = "0.6", default-features = false, features = ["cargo_bench_support"] } +criterion = { version = "4.0", package = "codspeed-criterion-compat" } neqo-transport = { path = ".", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } From 478ad4dc1aa038449f5371aa25f8af2ed00fde96 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 24 Oct 2025 13:53:03 +0300 Subject: [PATCH 02/28] Minimize --- Cargo.lock | 85 --------------------------------------- neqo-bin/Cargo.toml | 5 ++- neqo-common/Cargo.toml | 4 +- neqo-http3/Cargo.toml | 4 +- neqo-transport/Cargo.toml | 4 +- 5 files changed, 13 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b4905aa11..e0a91e90fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -294,8 +294,6 @@ dependencies = [ "num-traits", "once_cell", "oorandom", - "plotters", - "rayon", "regex", "serde", "serde_derive", @@ -325,31 +323,6 @@ dependencies = [ "itertools", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - [[package]] name = "crunchy" version = "0.2.2" @@ -1117,34 +1090,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "proc-macro2" version = "1.0.86" @@ -1195,26 +1140,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rayon" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "ref-cast" version = "1.0.24" @@ -1696,16 +1621,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "web-sys" -version = "0.3.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index 334a7335a4..35e3bed819 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -52,7 +52,10 @@ tokio = { version = "1", default-features = false, features = ["net", "time", "m url = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", features = ["async_tokio"] } +criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ + "async_tokio", + "cargo_bench_support", +] } neqo-bin = { path = ".", features = ["draft-29"] } neqo-http3 = { path = "./../neqo-http3", features = ["draft-29"] } neqo-transport = { path = "./../neqo-transport", features = ["draft-29"] } diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml index d4fcedfd5b..bef4b47c84 100644 --- a/neqo-common/Cargo.toml +++ b/neqo-common/Cargo.toml @@ -28,7 +28,9 @@ thiserror = { workspace = true } windows = { workspace = true, features = ["Win32_Media"] } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat" } +criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ + "cargo_bench_support", +] } neqo-crypto = { path = "../neqo-crypto" } test-fixture = { path = "../test-fixture" } regex = { workspace = true } diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index adea4a14ac..c16399cf08 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -33,7 +33,9 @@ thiserror = { workspace = true } url = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat" } +criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ + "cargo_bench_support", +] } neqo-http3 = { path = ".", features = ["draft-29"] } neqo-transport = { path = "./../neqo-transport", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml index 8d97338be8..50e743ee91 100644 --- a/neqo-transport/Cargo.toml +++ b/neqo-transport/Cargo.toml @@ -32,7 +32,9 @@ strum = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat" } +criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ + "cargo_bench_support", +] } neqo-transport = { path = ".", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } From 8fdf449247668587830bff837dfe9fcb5f5eeeec Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 24 Oct 2025 14:01:33 +0300 Subject: [PATCH 03/28] .github/workflows/codspeed.yml --- .github/workflows/codspeed.yml | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/codspeed.yml diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000000..19b45e7911 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,54 @@ +name: CodSpeed Benchmarks +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + merge_group: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 +# RUST_TEST_TIME_UNIT: 10,30 +# RUST_TEST_TIME_INTEGRATION: 10,30 +# RUST_TEST_TIME_DOCTEST: 10,30 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - uses: ./.github/actions/rust + with: + version: stable + token: ${{ secrets.GITHUB_TOKEN }} + + - id: nss-version + run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT" + + - uses: ./.github/actions/nss + with: + minimum-version: ${{ steps.nss-version.outputs.minimum }} + + - name: Run the benchmarks + uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 + with: + mode: instrumentation + run: cargo codspeed run --features bench + token: ${{ secrets.CODSPEED_TOKEN }} From fdaf47f42edf3f1b69293deff33a711b75d8bf33 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 10:30:35 +0200 Subject: [PATCH 04/28] Install cargo-codspeed --- .github/workflows/codspeed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 19b45e7911..96096fa069 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -37,6 +37,7 @@ jobs: - uses: ./.github/actions/rust with: version: stable + tools: cargo-codspeed token: ${{ secrets.GITHUB_TOKEN }} - id: nss-version From 1582a73f5b16202fd8030621ba0f4b1e7698dee2 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 11:49:40 +0200 Subject: [PATCH 05/28] Fixes --- .github/workflows/codspeed.yml | 7 ++++++- neqo-bin/Cargo.toml | 1 - neqo-common/Cargo.toml | 4 +--- neqo-http3/Cargo.toml | 4 +--- neqo-http3/benches/streams.rs | 2 +- neqo-transport/Cargo.toml | 4 +--- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 96096fa069..eabd90e4e3 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -47,9 +47,14 @@ jobs: with: minimum-version: ${{ steps.nss-version.outputs.minimum }} + - run: | + for crate in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name'); do + [ -e "$crate/benches" ] && cargo codspeed build --package "$crate" --locked --features bench + done + - name: Run the benchmarks uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 with: mode: instrumentation - run: cargo codspeed run --features bench + run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index 35e3bed819..14d674587a 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -54,7 +54,6 @@ url = { workspace = true } [dev-dependencies] criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ "async_tokio", - "cargo_bench_support", ] } neqo-bin = { path = ".", features = ["draft-29"] } neqo-http3 = { path = "./../neqo-http3", features = ["draft-29"] } diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml index bef4b47c84..e44ce4191b 100644 --- a/neqo-common/Cargo.toml +++ b/neqo-common/Cargo.toml @@ -28,9 +28,7 @@ thiserror = { workspace = true } windows = { workspace = true, features = ["Win32_Media"] } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ - "cargo_bench_support", -] } +criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false } neqo-crypto = { path = "../neqo-crypto" } test-fixture = { path = "../test-fixture" } regex = { workspace = true } diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index c16399cf08..b9379e5a4a 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -33,9 +33,7 @@ thiserror = { workspace = true } url = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ - "cargo_bench_support", -] } +criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false } neqo-http3 = { path = ".", features = ["draft-29"] } neqo-transport = { path = "./../neqo-transport", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } diff --git a/neqo-http3/benches/streams.rs b/neqo-http3/benches/streams.rs index b77dabb498..2d52299061 100644 --- a/neqo-http3/benches/streams.rs +++ b/neqo-http3/benches/streams.rs @@ -63,7 +63,7 @@ fn criterion_benchmark(c: &mut Criterion) { group.finish(); } - Criterion::default().configure_from_args().final_summary(); + //FIXME: This fails with codspeed: Criterion::default().configure_from_args().final_summary(); } criterion_group!(benches, criterion_benchmark); diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml index 50e743ee91..778bf6bc4d 100644 --- a/neqo-transport/Cargo.toml +++ b/neqo-transport/Cargo.toml @@ -32,9 +32,7 @@ strum = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ - "cargo_bench_support", -] } +criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false } neqo-transport = { path = ".", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } From 98f3e2579f7c893e5dc1f295261eb2db3564b2e9 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 11:57:43 +0200 Subject: [PATCH 06/28] Fixes --- .github/workflows/codspeed.yml | 4 +++- neqo-common/benches/decoder.rs | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index eabd90e4e3..6e692d68ad 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -49,7 +49,9 @@ jobs: - run: | for crate in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name'); do - [ -e "$crate/benches" ] && cargo codspeed build --package "$crate" --locked --features bench + if [ -e "$crate/benches" ]; then + cargo codspeed build --package "$crate" --locked --features bench + fi done - name: Run the benchmarks diff --git a/neqo-common/benches/decoder.rs b/neqo-common/benches/decoder.rs index d3edc3e2d2..2ec8935c9f 100644 --- a/neqo-common/benches/decoder.rs +++ b/neqo-common/benches/decoder.rs @@ -5,6 +5,10 @@ // except according to those terms. #![expect(clippy::unwrap_used, reason = "OK in a bench.")] +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] use std::hint::black_box; From bd2af175d6b19ae29e5f19eebbf66b76d1962601 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 13:06:07 +0200 Subject: [PATCH 07/28] Fixes --- neqo-bin/benches/main.rs | 4 ++++ neqo-http3/benches/streams.rs | 5 +++++ neqo-transport/benches/range_tracker.rs | 5 +++++ neqo-transport/benches/rx_stream_orderer.rs | 5 +++++ neqo-transport/benches/sent_packets.rs | 5 +++++ neqo-transport/benches/transfer.rs | 5 +++++ 6 files changed, 29 insertions(+) diff --git a/neqo-bin/benches/main.rs b/neqo-bin/benches/main.rs index f8ddaa75dc..163d6d1af3 100644 --- a/neqo-bin/benches/main.rs +++ b/neqo-bin/benches/main.rs @@ -5,6 +5,10 @@ // except according to those terms. #![expect(clippy::unwrap_used, reason = "OK in a bench.")] +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] use std::{env, hint::black_box, net::SocketAddr, path::PathBuf, str::FromStr as _}; diff --git a/neqo-http3/benches/streams.rs b/neqo-http3/benches/streams.rs index 2d52299061..19873e6893 100644 --- a/neqo-http3/benches/streams.rs +++ b/neqo-http3/benches/streams.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::{hint::black_box, time::Duration}; use criterion::{criterion_group, criterion_main, Criterion, Throughput}; diff --git a/neqo-transport/benches/range_tracker.rs b/neqo-transport/benches/range_tracker.rs index 02094f1fa4..93bb2b3abb 100644 --- a/neqo-transport/benches/range_tracker.rs +++ b/neqo-transport/benches/range_tracker.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::hint::black_box; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/neqo-transport/benches/rx_stream_orderer.rs b/neqo-transport/benches/rx_stream_orderer.rs index c4a73fb85d..8e01e283f8 100644 --- a/neqo-transport/benches/rx_stream_orderer.rs +++ b/neqo-transport/benches/rx_stream_orderer.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::hint::black_box; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/neqo-transport/benches/sent_packets.rs b/neqo-transport/benches/sent_packets.rs index d6c812f893..8315f136d8 100644 --- a/neqo-transport/benches/sent_packets.rs +++ b/neqo-transport/benches/sent_packets.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::{hint::black_box, time::Instant}; use criterion::{criterion_group, criterion_main, Criterion}; diff --git a/neqo-transport/benches/transfer.rs b/neqo-transport/benches/transfer.rs index 1fbc2c1b30..2c48a90988 100644 --- a/neqo-transport/benches/transfer.rs +++ b/neqo-transport/benches/transfer.rs @@ -4,6 +4,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + use std::{hint::black_box, time::Duration}; use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion}; From a4a7178fbcddcbe635641cf0d5bd3a8dedcc07f7 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 13:22:34 +0200 Subject: [PATCH 08/28] Remove comment --- .github/workflows/codspeed.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 6e692d68ad..3297eeab3f 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -10,9 +10,6 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 -# RUST_TEST_TIME_UNIT: 10,30 -# RUST_TEST_TIME_INTEGRATION: 10,30 -# RUST_TEST_TIME_DOCTEST: 10,30 concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} From 62a2a61c0867691ed2ba3c42eec59dde26c642a3 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 15:40:12 +0200 Subject: [PATCH 09/28] Use different measurement modes --- .github/workflows/codspeed.yml | 7 ++++++- neqo-bin/Cargo.toml | 3 +++ neqo-http3/Cargo.toml | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 3297eeab3f..5a6bdf77cc 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -47,7 +47,12 @@ jobs: - run: | for crate in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name'); do if [ -e "$crate/benches" ]; then - cargo codspeed build --package "$crate" --locked --features bench + for bench in $(find "$crate/benches" -depth 1 -print0 -iname "*.rs" | xargs -0 -n1 basename -s .rs); do + # Set measurement mode from Cargo.toml metadata, default to "instrumentation". + mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') + echo "Building benchmark '$bench' in crate '$crate' with measurement mode '$mode'" + cargo codspeed build --package "$crate" --locked --features bench --bench "$bench" --release --measurement-mode "$mode" + done fi done diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index 14d674587a..79f64994ff 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -72,6 +72,9 @@ ignored = ["log"] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options bench = false +[package.metadata.bench.main] +codspeed.mode = "wallclock" # Use "wallclock" mode for "main" bench with codspeed. + [[bench]] name = "main" harness = false diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index b9379e5a4a..6f8bd49818 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -56,6 +56,9 @@ ignored = ["criterion", "log"] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options bench = false +[package.metadata.bench.streams] +codspeed.mode = "wallclock" # Use "wallclock" mode for "main" bench with codspeed. + [[bench]] name = "streams" harness = false From f22b93b01971f7ec521b8720aaf7715adc61ffbf Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 15:59:30 +0200 Subject: [PATCH 10/28] Fix --- .github/workflows/codspeed.yml | 3 ++- neqo-bin/Cargo.toml | 2 +- neqo-http3/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 5a6bdf77cc..aeefa316bc 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -47,7 +47,8 @@ jobs: - run: | for crate in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name'); do if [ -e "$crate/benches" ]; then - for bench in $(find "$crate/benches" -depth 1 -print0 -iname "*.rs" | xargs -0 -n1 basename -s .rs); do + for bench in "$crate/benches"/*.rs; do + bench=$(basename -s .rs "$bench") # Set measurement mode from Cargo.toml metadata, default to "instrumentation". mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') echo "Building benchmark '$bench' in crate '$crate' with measurement mode '$mode'" diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index 79f64994ff..482107ada2 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -73,7 +73,7 @@ ignored = ["log"] bench = false [package.metadata.bench.main] -codspeed.mode = "wallclock" # Use "wallclock" mode for "main" bench with codspeed. +codspeed.mode = "walltime" # Use "walltime" mode for "main" bench with codspeed. [[bench]] name = "main" diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index 6f8bd49818..f4dc71468a 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -57,7 +57,7 @@ ignored = ["criterion", "log"] bench = false [package.metadata.bench.streams] -codspeed.mode = "wallclock" # Use "wallclock" mode for "main" bench with codspeed. +codspeed.mode = "walltime" # Use "walltime" mode for "main" bench with codspeed. [[bench]] name = "streams" From 8638d289f4323771f3c7e377bcec74e323782c14 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 16:20:45 +0200 Subject: [PATCH 11/28] Update .github/workflows/codspeed.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Lars Eggert --- .github/workflows/codspeed.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index aeefa316bc..6aee0de872 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -60,6 +60,5 @@ jobs: - name: Run the benchmarks uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 with: - mode: instrumentation run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} From d97578da8092c9d57439a3191dd4f6158cb95a3f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 16:21:14 +0200 Subject: [PATCH 12/28] Update .github/workflows/codspeed.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Lars Eggert --- .github/workflows/codspeed.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 6aee0de872..01801fd4fe 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -45,17 +45,18 @@ jobs: minimum-version: ${{ steps.nss-version.outputs.minimum }} - run: | - for crate in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name'); do - if [ -e "$crate/benches" ]; then - for bench in "$crate/benches"/*.rs; do + while IFS=: read -r crate manifest_path; do + dir=$(dirname "$manifest_path") + if [ -e "$dir/benches" ]; then + for bench in "$dir/benches"/*.rs; do bench=$(basename -s .rs "$bench") # Set measurement mode from Cargo.toml metadata, default to "instrumentation". mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') - echo "Building benchmark '$bench' in crate '$crate' with measurement mode '$mode'" + echo "Building benchmark '$bench' in crate '$crate' (dir '$dir') with measurement mode '$mode'" cargo codspeed build --package "$crate" --locked --features bench --bench "$bench" --release --measurement-mode "$mode" done fi - done + done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') - name: Run the benchmarks uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 From bf7c12316aa39d2e905a37989150a6b67a141377 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 29 Oct 2025 16:32:17 +0200 Subject: [PATCH 13/28] Fix --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 01801fd4fe..b8ef3305b5 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -53,7 +53,7 @@ jobs: # Set measurement mode from Cargo.toml metadata, default to "instrumentation". mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') echo "Building benchmark '$bench' in crate '$crate' (dir '$dir') with measurement mode '$mode'" - cargo codspeed build --package "$crate" --locked --features bench --bench "$bench" --release --measurement-mode "$mode" + cargo codspeed build --package "$crate" --locked --features bench --bench "$bench" --measurement-mode "$mode" done fi done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') From 0cdb59745b913707865308fb43041df487b49685 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 10:36:13 +0200 Subject: [PATCH 14/28] Shard benches --- .github/workflows/codspeed.yml | 64 ++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index b8ef3305b5..6c96c4e506 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -23,9 +23,50 @@ defaults: shell: bash jobs: + bench-matrix: + name: Determine benchmark matrix + runs-on: ubuntu-24.04 + outputs: + benches: ${{ steps.benches.outputs.benches }} + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + sparse-checkout: Cargo.toml + persist-credentials: false + - id: benches + run: | + # Create a GitHub matrix where each entry is a bench with its + # associated crate and benchmarking mode taken from the Cargo + # metadata. + echo "benches=[" > benches.json + first=true + while IFS=: read -r crate manifest_path; do + dir=$(dirname "$manifest_path") + if [ -e "$dir/benches" ]; then + for bench in "$dir/benches"/*.rs; do + bench=$(basename -s .rs "$bench") + # Set measurement mode from Cargo.toml metadata, default to "instrumentation". + mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') + if [ "$first" = true ]; then + first=false + else + echo "," >> benches.json + fi + echo "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" >> benches.json + done + fi + done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') + echo "]" >> benches.json + jq benches.json | tee >> "$GITHUB_OUTPUT" + benchmarks: name: Run benchmarks - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + strategy: + matrix: + bench: ${{ fromJson(needs.bench-matrix.outputs.benches) }} + needs: bench-matrix + steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: @@ -44,22 +85,17 @@ jobs: with: minimum-version: ${{ steps.nss-version.outputs.minimum }} - - run: | - while IFS=: read -r crate manifest_path; do - dir=$(dirname "$manifest_path") - if [ -e "$dir/benches" ]; then - for bench in "$dir/benches"/*.rs; do - bench=$(basename -s .rs "$bench") - # Set measurement mode from Cargo.toml metadata, default to "instrumentation". - mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') - echo "Building benchmark '$bench' in crate '$crate' (dir '$dir') with measurement mode '$mode'" - cargo codspeed build --package "$crate" --locked --features bench --bench "$bench" --measurement-mode "$mode" - done - fi - done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') + - env: + BENCH: ${{ matrix.bench.bench }} + CRATE: ${{ matrix.bench.crate }} + MODE: ${{ matrix.bench.mode }} + run: | + echo "Building benchmark '$BENCH' in crate '$CRATE' with measurement mode '$MODE'" + cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "$MODE" - name: Run the benchmarks uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 with: + mode: ${{ matrix.bench.mode }} run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} From 2f72a0d170deaa040bc6a763fa522a068f720761 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 10:41:04 +0200 Subject: [PATCH 15/28] No jq --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 6c96c4e506..015692a81d 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -57,7 +57,7 @@ jobs: fi done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') echo "]" >> benches.json - jq benches.json | tee >> "$GITHUB_OUTPUT" + tee benches.json >> "$GITHUB_OUTPUT" benchmarks: name: Run benchmarks From 67e2f4942644d32b8609d0ac915290dfedb07268 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 10:45:40 +0200 Subject: [PATCH 16/28] Rust --- .github/workflows/codspeed.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 015692a81d..3d77a3925d 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -33,6 +33,10 @@ jobs: with: sparse-checkout: Cargo.toml persist-credentials: false + - uses: ./.github/actions/rust + with: + version: stable + token: ${{ secrets.GITHUB_TOKEN }} - id: benches run: | # Create a GitHub matrix where each entry is a bench with its From 0eb940299e85e25a522bc56a1d58e21dfa756236 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 10:47:04 +0200 Subject: [PATCH 17/28] Update .github/workflows/codspeed.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Lars Eggert --- .github/workflows/codspeed.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 3d77a3925d..cd670a6427 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -61,7 +61,8 @@ jobs: fi done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') echo "]" >> benches.json - tee benches.json >> "$GITHUB_OUTPUT" + benches=$(cat benches.json) + echo "benches=$benches" >> "$GITHUB_OUTPUT" benchmarks: name: Run benchmarks From 1ffe3e93424863b67f6e2993b4126fc8a7616cab Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 10:50:22 +0200 Subject: [PATCH 18/28] Again --- .github/workflows/codspeed.yml | 48 ++++++++++++++++------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index cd670a6427..5cabeb7ce0 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -31,38 +31,34 @@ jobs: steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: - sparse-checkout: Cargo.toml persist-credentials: false - - uses: ./.github/actions/rust - with: - version: stable - token: ${{ secrets.GITHUB_TOKEN }} - id: benches run: | # Create a GitHub matrix where each entry is a bench with its # associated crate and benchmarking mode taken from the Cargo # metadata. - echo "benches=[" > benches.json - first=true - while IFS=: read -r crate manifest_path; do - dir=$(dirname "$manifest_path") - if [ -e "$dir/benches" ]; then - for bench in "$dir/benches"/*.rs; do - bench=$(basename -s .rs "$bench") - # Set measurement mode from Cargo.toml metadata, default to "instrumentation". - mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') - if [ "$first" = true ]; then - first=false - else - echo "," >> benches.json - fi - echo "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" >> benches.json - done - fi - done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') - echo "]" >> benches.json - benches=$(cat benches.json) - echo "benches=$benches" >> "$GITHUB_OUTPUT" + { + echo "benches=[" + first=true + while IFS=: read -r crate manifest_path; do + dir=$(dirname "$manifest_path") + if [ -e "$dir/benches" ]; then + for bench in "$dir/benches"/*.rs; do + bench=$(basename -s .rs "$bench") + # Set measurement mode from Cargo.toml metadata, default to "instrumentation". + mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') + if [ "$first" = true ]; then + first=false + else + echo "," + fi + echo "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" + done + fi + done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') + echo "]" + } > benches.json + tee benches.json >> "$GITHUB_OUTPUT" benchmarks: name: Run benchmarks From 11598b91e0601abfe213fcd49d14738257da9fa1 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 10:55:55 +0200 Subject: [PATCH 19/28] cat --- .github/workflows/codspeed.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 5cabeb7ce0..4a42ea22f5 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -58,7 +58,8 @@ jobs: done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') echo "]" } > benches.json - tee benches.json >> "$GITHUB_OUTPUT" + cat benches.json + cat benches.json >> "$GITHUB_OUTPUT" benchmarks: name: Run benchmarks From ab3a71a77801e2300af88b3e9651afeee62f68ba Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 10:56:24 +0200 Subject: [PATCH 20/28] Update neqo-http3/Cargo.toml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Lars Eggert --- neqo-http3/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index f4dc71468a..1ac822839e 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -57,7 +57,7 @@ ignored = ["criterion", "log"] bench = false [package.metadata.bench.streams] -codspeed.mode = "walltime" # Use "walltime" mode for "main" bench with codspeed. +codspeed.mode = "walltime" # Use "walltime" mode for "streams" bench with codspeed. [[bench]] name = "streams" From 624f7a0e7a6e9ea160cdb68a735ad9fc26856146 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 11:01:20 +0200 Subject: [PATCH 21/28] Again --- .github/workflows/codspeed.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 4a42ea22f5..911644ac4a 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -38,7 +38,7 @@ jobs: # associated crate and benchmarking mode taken from the Cargo # metadata. { - echo "benches=[" + echo "[" first=true while IFS=: read -r crate manifest_path; do dir=$(dirname "$manifest_path") @@ -57,9 +57,9 @@ jobs: fi done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') echo "]" - } > benches.json + } | jq --raw-input --compact-output --slurp > benches.json cat benches.json - cat benches.json >> "$GITHUB_OUTPUT" + echo "benches=$(cat benches.json)" >> "${GITHUB_OUTPUT}" benchmarks: name: Run benchmarks From 743369f14e132cf2b6b575fd3d6186949ce0299e Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 11:06:31 +0200 Subject: [PATCH 22/28] Again --- .github/workflows/codspeed.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 911644ac4a..4803cfcc8f 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -38,7 +38,7 @@ jobs: # associated crate and benchmarking mode taken from the Cargo # metadata. { - echo "[" + echo -n "[" first=true while IFS=: read -r crate manifest_path; do dir=$(dirname "$manifest_path") @@ -50,14 +50,14 @@ jobs: if [ "$first" = true ]; then first=false else - echo "," + echo -n "," fi - echo "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" + echo -n "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" done fi done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') - echo "]" - } | jq --raw-input --compact-output --slurp > benches.json + echo -n "]" + } > benches.json cat benches.json echo "benches=$(cat benches.json)" >> "${GITHUB_OUTPUT}" From b0a57e70f6e464c5ae61cabc50728477b201006b Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 11:09:22 +0200 Subject: [PATCH 23/28] Update .github/workflows/codspeed.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Lars Eggert --- .github/workflows/codspeed.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 4803cfcc8f..56babfb302 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -46,6 +46,10 @@ jobs: for bench in "$dir/benches"/*.rs; do bench=$(basename -s .rs "$bench") # Set measurement mode from Cargo.toml metadata, default to "instrumentation". + # The following jq pipeline: + # - Selects the package with name == $crate + # - Looks up .metadata.bench[$bench].codspeed.mode for the benchmark + # - If not set, defaults to "instrumentation" mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') if [ "$first" = true ]; then first=false From 46331381299e51dc223d0e755b3d554959e6051e Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 30 Oct 2025 11:23:04 +0200 Subject: [PATCH 24/28] Fixes --- .github/workflows/codspeed.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 56babfb302..67a681004c 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -1,4 +1,4 @@ -name: CodSpeed Benchmarks +name: CodSpeed on: push: branches: ["main"] @@ -24,7 +24,7 @@ defaults: jobs: bench-matrix: - name: Determine benchmark matrix + name: Determine bench matrix runs-on: ubuntu-24.04 outputs: benches: ${{ steps.benches.outputs.benches }} @@ -38,7 +38,7 @@ jobs: # associated crate and benchmarking mode taken from the Cargo # metadata. { - echo -n "[" + echo -n "benches=[" first=true while IFS=: read -r crate manifest_path; do dir=$(dirname "$manifest_path") @@ -51,6 +51,8 @@ jobs: # - Looks up .metadata.bench[$bench].codspeed.mode for the benchmark # - If not set, defaults to "instrumentation" mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') + # FIXME: Always use "instrumentation" mode to eliminate CI VM effects. + mode="instrumentation" if [ "$first" = true ]; then first=false else @@ -61,12 +63,10 @@ jobs: fi done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') echo -n "]" - } > benches.json - cat benches.json - echo "benches=$(cat benches.json)" >> "${GITHUB_OUTPUT}" + } >> "${GITHUB_OUTPUT}" benchmarks: - name: Run benchmarks + name: Run bench runs-on: ubuntu-24.04 strategy: matrix: @@ -91,15 +91,14 @@ jobs: with: minimum-version: ${{ steps.nss-version.outputs.minimum }} - - env: + - name: Build bench + env: BENCH: ${{ matrix.bench.bench }} CRATE: ${{ matrix.bench.crate }} MODE: ${{ matrix.bench.mode }} - run: | - echo "Building benchmark '$BENCH' in crate '$CRATE' with measurement mode '$MODE'" - cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "$MODE" + run: cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "$MODE" - - name: Run the benchmarks + - name: Run bench & upload results uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 with: mode: ${{ matrix.bench.mode }} From ce344ef3e989fc1dd0461d7781d45cbcce65f0f9 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 7 Nov 2025 16:03:03 -0500 Subject: [PATCH 25/28] Bump codspeed --- Cargo.lock | 113 +++++++------------------------------- neqo-bin/Cargo.toml | 2 +- neqo-common/Cargo.toml | 2 +- neqo-http3/Cargo.toml | 2 +- neqo-transport/Cargo.toml | 2 +- 5 files changed, 23 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26e0b90e15..18b863414e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,12 +112,6 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - [[package]] name = "cast" version = "0.3.0" @@ -245,27 +239,27 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "codspeed" -version = "4.0.5" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe68fdd3fe25bc26de0230718d74eb150f09f70be3141c61ea7f9b00812054aa" +checksum = "c3b847e05a34be5c38f3f2a5052178a3bd32e6b5702f3ea775efde95c483a539" dependencies = [ "anyhow", "cc", "colored", + "getrandom 0.2.16", "glob", "libc", "nix", "serde", "serde_json", "statrs", - "uuid", ] [[package]] name = "codspeed-criterion-compat" -version = "4.0.5" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a06533c3c2c7b43b2efcca2dc0f228097b4c582c95f59cc187bbeb926b42cfb" +checksum = "30a0e2a53beb18dec493ec133f226e0d35e8bb2fdc638ccf7351696eabee416c" dependencies = [ "clap", "codspeed", @@ -278,9 +272,9 @@ dependencies = [ [[package]] name = "codspeed-criterion-compat-walltime" -version = "4.0.5" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4102bd4dcfb09fae5cf5ca86c6c5ad0bd0cabe834cd84b5e7b7dd969fb9093e8" +checksum = "1f652d6e6d40bba0f5c244744db94d92a26b7f083df18692df88fb0772f1c793" dependencies = [ "anes", "cast", @@ -524,6 +518,17 @@ dependencies = [ "test-fixture", ] +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.3.3" @@ -775,20 +780,10 @@ version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ - "getrandom", + "getrandom 0.3.3", "libc", ] -[[package]] -name = "js-sys" -version = "0.3.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -1525,17 +1520,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "uuid" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" -dependencies = [ - "getrandom", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "walkdir" version = "2.3.2" @@ -1562,65 +1546,6 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" -dependencies = [ - "unicode-ident", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index 482107ada2..81fa6d95e6 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -52,7 +52,7 @@ tokio = { version = "1", default-features = false, features = ["net", "time", "m url = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false, features = [ +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false, features = [ "async_tokio", ] } neqo-bin = { path = ".", features = ["draft-29"] } diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml index e44ce4191b..33c064ef53 100644 --- a/neqo-common/Cargo.toml +++ b/neqo-common/Cargo.toml @@ -28,7 +28,7 @@ thiserror = { workspace = true } windows = { workspace = true, features = ["Win32_Media"] } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false } +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false } neqo-crypto = { path = "../neqo-crypto" } test-fixture = { path = "../test-fixture" } regex = { workspace = true } diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index 1ac822839e..233619c197 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -33,7 +33,7 @@ thiserror = { workspace = true } url = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false } +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false } neqo-http3 = { path = ".", features = ["draft-29"] } neqo-transport = { path = "./../neqo-transport", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml index 778bf6bc4d..4be52ab21d 100644 --- a/neqo-transport/Cargo.toml +++ b/neqo-transport/Cargo.toml @@ -32,7 +32,7 @@ strum = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -criterion = { version = "4.0", package = "codspeed-criterion-compat", default-features = false } +criterion = { version = "4", package = "codspeed-criterion-compat", default-features = false } neqo-transport = { path = ".", features = ["draft-29"] } test-fixture = { path = "../test-fixture" } From 46ace57c68aec01a6e33876986ce8b2ba415713e Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 7 Nov 2025 16:08:22 -0500 Subject: [PATCH 26/28] Merge --- Cargo.lock | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84ef288e2c..cf06742958 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -272,14 +272,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "criterion-plot" -version = "0.6.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", "itertools", @@ -718,7 +718,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -746,6 +746,12 @@ dependencies = [ "libc", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.175" From 7aeece4e8928183fc21fc6b46d823330c10d2c67 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 11 Nov 2025 15:37:36 +0200 Subject: [PATCH 27/28] Remove FIXME --- neqo-http3/benches/streams.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/neqo-http3/benches/streams.rs b/neqo-http3/benches/streams.rs index 19873e6893..c9ff4c6ac3 100644 --- a/neqo-http3/benches/streams.rs +++ b/neqo-http3/benches/streams.rs @@ -67,8 +67,6 @@ fn criterion_benchmark(c: &mut Criterion) { group.finish(); } - - //FIXME: This fails with codspeed: Criterion::default().configure_from_args().final_summary(); } criterion_group!(benches, criterion_benchmark); From 4f9e5e63ebcbacd905d72f70061ff5e372d13c79 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 11 Nov 2025 15:42:32 +0200 Subject: [PATCH 28/28] Cargo.lock --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 63f13e5491..79bc128c5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,7 +1197,6 @@ dependencies = [ "memchr", "ryu", "serde", - "serde_core", ] [[package]]