From 79bbe354825c30757bb625e57ed65e858a735a86 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Mon, 8 Jun 2026 08:02:41 +0000 Subject: [PATCH 01/29] README includes install instructions --- README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d9f2332..81a08ea 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,41 @@ Automatically apply decoupling and partitioning to Hydro programs for higher throughput. -> ![NOTE] +> [!NOTE] > Only Linux is supported, as we compile with `glibc` for better performance and more legibile `perf` results. ## Installation +### Permissions +The Linux machine on which you are running hydro-optimize will need permissions to launch VMs. If this is your local machine, you can simply sign into your AWS account locally; otherwise you will need to grant the remote machine the appropriate permissions. + +#### AWS EC2 +The launch instance types are specified in `deploy.rs`. +Since we use `glibc` for compilation, we need to launch from a machine with the same `glibc` version as the destination, which currently is an Amazon Linux 2023 instance. +Do not use the latest Ubuntu machine to launch for this reason. +1. Go to AWS IAM Roles > Create role. +2. Select "EC2" as the Use case. +3. Add "AmazonEC2FullAccess" as the Permission policy. +4. Give it a name (I named it "EC2FullRole"). +5. Go to the Instance you are running hydro-optimize from, Actions > Security > Modify IAM role. +6. Give it the EC2FullRole. + +#### GCP Compute Engine +TODO + +### Terraform +Terraform is used to spin up machines. +```bash +sudo apt-get update && sudo apt-get install -y gnupg software-properties-common +wget -O- https://apt.releases.hashicorp.com/gpg | \ +gpg --dearmor | \ +sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(grep -oP '(?<=UBUNTU_CODENAME=).*' /etc/os-release || lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list +sudo apt update +sudo apt-get install terraform +``` + ### ILP Solver We rely on the [Gurobi](https://www.gurobi.com/) ILP solver to find the optimal set of rewrites. Gurobi is not free; you will need to either create a Gurobi account (and use a free educational license) or use your organization's license, if available. From 2636e5daf7cc7454af4785975e6ca7db0c5dfff8 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 9 Jun 2026 09:30:27 +0000 Subject: [PATCH 02/29] Network calibration without restarting for different sizes, more Amazon Linux instructions --- Cargo.lock | 167 +++++++++--------- Cargo.toml | 15 +- README.md | 21 ++- hydro_optimize/src/deploy_and_analyze.rs | 54 ++++-- .../examples/benchmark_paxos.rs | 2 +- hydro_optimize_examples/examples/cas.rs | 2 +- .../examples/network_calibrator.rs | 92 +++++----- setup_gurobi.sh | 2 +- 8 files changed, 188 insertions(+), 167 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5f6315..103d149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,9 +369,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "backtrace" @@ -440,9 +440,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" dependencies = [ "serde_core", ] @@ -628,9 +628,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "by_address" @@ -748,9 +748,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.62" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "shlex", @@ -798,9 +798,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -949,7 +949,7 @@ dependencies = [ [[package]] name = "copy_span" version = "0.1.1" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "proc-macro2", "quote", @@ -1186,7 +1186,7 @@ dependencies = [ [[package]] name = "dfir_lang" version = "0.16.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "auto_impl", "documented", @@ -1204,7 +1204,7 @@ dependencies = [ [[package]] name = "dfir_pipes" version = "0.0.1" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "futures-core", "futures-sink", @@ -1220,7 +1220,7 @@ dependencies = [ [[package]] name = "dfir_rs" version = "0.16.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "bincode", "bytes", @@ -1283,9 +1283,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -1765,9 +1765,9 @@ dependencies = [ [[package]] name = "good_lp" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf53d2a05420f562c917615b80018647ca03766aa58376de077034627da0fb10" +checksum = "745190412d5ff4a54335cd16229a475ad3fb8f5474a5c1358292d62932187ea7" dependencies = [ "fnv", "lp-solvers", @@ -1876,9 +1876,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1916,7 +1916,7 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "hydro_build_utils" version = "0.1.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "insta", "rustc_version", @@ -1925,7 +1925,7 @@ dependencies = [ [[package]] name = "hydro_deploy" version = "0.16.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "anyhow", "append-only-vec", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "hydro_deploy_integration" version = "0.16.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "async-recursion", "async-trait", @@ -1980,7 +1980,7 @@ dependencies = [ [[package]] name = "hydro_lang" version = "0.16.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "auto_impl", "backtrace", @@ -2081,7 +2081,7 @@ dependencies = [ [[package]] name = "hydro_std" version = "0.16.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "hdrhistogram", "hydro_lang", @@ -2093,7 +2093,7 @@ dependencies = [ [[package]] name = "hydro_test" version = "0.0.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "bytes", "colored", @@ -2114,9 +2114,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -2307,7 +2307,7 @@ dependencies = [ [[package]] name = "include_mdtests" version = "0.0.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "glob", "proc-macro2", @@ -2471,20 +2471,19 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] [[package]] name = "lattices" version = "0.7.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "cc-traits", "fst", @@ -2500,7 +2499,7 @@ dependencies = [ [[package]] name = "lattices_macro" version = "0.6.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "hydro_build_utils", "proc-macro-crate 3.5.0", @@ -2548,9 +2547,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ "libc", ] @@ -2611,9 +2610,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lp-solvers" @@ -2681,9 +2680,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "memmap2" @@ -2718,9 +2717,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -3331,7 +3330,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.11+spec-1.1.0", + "toml_edit 0.25.12+spec-1.1.0", ] [[package]] @@ -3815,9 +3814,9 @@ dependencies = [ [[package]] name = "russh-sftp" -version = "2.2.2" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c826d40e310bbcb377fd8f6c5873c12eaf54da30569d8e8da56914fdac9773cd" +checksum = "9ed8949eca4163c18a8f59ff96d32cf61e9c13b9735e21ef32b3907f4aafa1a9" dependencies = [ "bitflags", "bytes", @@ -4161,9 +4160,9 @@ checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -4200,7 +4199,7 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "sinktools" version = "0.1.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "futures-util", "pin-project-lite", @@ -4239,9 +4238,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -4727,9 +4726,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", "toml_datetime 1.1.1+spec-1.1.0", @@ -4922,9 +4921,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" @@ -4934,9 +4933,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -4992,9 +4991,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "js-sys", "wasm-bindgen", @@ -5009,7 +5008,7 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "variadics" version = "0.1.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "hashbrown 0.14.5", "hydro_build_utils", @@ -5019,7 +5018,7 @@ dependencies = [ [[package]] name = "variadics_macro" version = "0.7.0" -source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#20880da3faec5174cae96cfdaf472de8f0171d4c" +source = "git+https://github.com/davidchuyaya/hydroflow.git?branch=fix_network_pull_count#544d4c3501465a38e082cacb5e43393a0aa2f2a4" dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", @@ -5069,9 +5068,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -5082,9 +5081,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -5092,9 +5091,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5102,9 +5101,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -5115,9 +5114,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] @@ -5171,9 +5170,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", @@ -5200,9 +5199,9 @@ dependencies = [ [[package]] name = "which" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" +checksum = "c789537cf2f7f55be8e6192f92e464174ee55f91af622777f7f1ceb0dbccd03e" dependencies = [ "libc", ] @@ -5683,9 +5682,9 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -5706,18 +5705,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index b1522bf..a902229 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,12 +34,11 @@ regex = "1.11.1" # hydro_deploy = { path = "/local/home/chudc/hydroflow/hydro_deploy/core" } # include_mdtests = { path = "/local/home/chudc/hydroflow/include_mdtests" } -# Uncomment to use local hydro checkout on Mac # [patch."https://github.com/davidchuyaya/hydroflow.git"] -# hydro_lang = { path = "/home/chudc/hydroflow/hydro_lang" } -# hydro_std = { path = "/home/chudc/hydroflow/hydro_std" } -# hydro_test = { path = "/home/chudc/hydroflow/hydro_test" } -# dfir_lang = { path = "/home/chudc/hydroflow/dfir_lang" } -# hydro_build_utils = { path = "/home/chudc/hydroflow/hydro_build_utils" } -# hydro_deploy = { path = "/home/chudc/hydroflow/hydro_deploy/core" } -# include_mdtests = { path = "/home/chudc/hydroflow/include_mdtests" } +# hydro_lang = { path = "/home/ec2-user/hydroflow/hydro_lang" } +# hydro_std = { path = "/home/ec2-user/hydroflow/hydro_std" } +# hydro_test = { path = "/home/ec2-user/hydroflow/hydro_test" } +# dfir_lang = { path = "/home/ec2-user/hydroflow/dfir_lang" } +# hydro_build_utils = { path = "/home/ec2-user/hydroflow/hydro_build_utils" } +# hydro_deploy = { path = "/home/ec2-user/hydroflow/hydro_deploy/core" } +# include_mdtests = { path = "/home/ec2-user/hydroflow/include_mdtests" } diff --git a/README.md b/README.md index 81a08ea..1ea2d32 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,21 @@ Automatically apply decoupling and partitioning to Hydro programs for higher thr ## Installation +### Rust +Install Rust. Install `rust-analyzer` as well if you plan on editing the code in an IDE. +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source ~/.bashrc +rustup component add rust-analyzer +``` + ### Permissions The Linux machine on which you are running hydro-optimize will need permissions to launch VMs. If this is your local machine, you can simply sign into your AWS account locally; otherwise you will need to grant the remote machine the appropriate permissions. #### AWS EC2 The launch instance types are specified in `deploy.rs`. Since we use `glibc` for compilation, we need to launch from a machine with the same `glibc` version as the destination, which currently is an Amazon Linux 2023 instance. -Do not use the latest Ubuntu machine to launch for this reason. +Do not use another Linux machine to launch for this reason. 1. Go to AWS IAM Roles > Create role. 2. Select "EC2" as the Use case. 3. Add "AmazonEC2FullAccess" as the Permission policy. @@ -27,14 +35,11 @@ TODO ### Terraform Terraform is used to spin up machines. +These are instructions for Amazon Linux machines; refer to [the official site](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) for your architecture. ```bash -sudo apt-get update && sudo apt-get install -y gnupg software-properties-common -wget -O- https://apt.releases.hashicorp.com/gpg | \ -gpg --dearmor | \ -sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(grep -oP '(?<=UBUNTU_CODENAME=).*' /etc/os-release || lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list -sudo apt update -sudo apt-get install terraform +sudo dnf install -y dnf-plugins-core +sudo dnf config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo +sudo dnf -y install terraform ``` ### ILP Solver diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index 6b84a03..e5eaa89 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; use std::time::Duration; -use hydro_deploy::Deployment; +use hydro_deploy::{Deployment, HYDRO_NET_CALIBRATE_ENV}; use hydro_lang::compile::built::BuiltFlow; use hydro_lang::compile::deploy::DeployResult; use hydro_lang::compile::ir::{HydroNode, HydroRoot, deep_clone, traverse_dfir}; @@ -586,6 +586,7 @@ pub struct BenchmarkArgs { pub aws: bool, } +#[derive(Clone)] pub struct BenchmarkConfig { pub name: String, /// Workload variant (e.g. "read_heavy"). Determines output subdirectory name. @@ -601,7 +602,7 @@ pub struct BenchmarkConfig { pub num_runs: usize, /// If set, enables network calibration mode: sets `HYDRO_NET_CALIBRATE=` /// on all hosts and forces single-host deployment. - pub calibrate_message_size: Option, + pub calibrate_message_sizes: Option>, } pub const START_MEASUREMENT_SECOND: usize = 30; @@ -865,7 +866,7 @@ async fn run_scaling_loop<'a>( final_run_metadata = run_metadata; if matches!(optimizations.kind, OptimizationKind::SizeAnalysis) - || config.calibrate_message_size.is_some() + || config.calibrate_message_sizes.is_some() { break 'outer; } @@ -1167,8 +1168,7 @@ pub async fn benchmark_protocol<'a>( "Must run at least one iteration of the benchmark" ); - if let Some(size) = config.calibrate_message_size { - reusable_hosts.insert_env("HYDRO_NET_CALIBRATE".to_string(), size.to_string()); + if config.calibrate_message_sizes.is_some() { reusable_hosts.set_single_host(true); } @@ -1189,17 +1189,39 @@ pub async fn benchmark_protocol<'a>( _ => panic!("Explicit optimization kinds are no longer allowed as parameters"), }; - let final_run_metadata = run_scaling_loop( - &mut reusable_hosts, - &mut deployment, - &built, - &config, - &Optimizations::default(), - &HashMap::new(), - None, - &HashSet::new(), - ) - .await; + // Run a loop for each network calibration size, if provided + let final_run_metadata = if let Some(ref sizes) = config.calibrate_message_sizes { + let mut last_metadata = RunMetadata::default(); + for &size in sizes { + reusable_hosts.insert_env(HYDRO_NET_CALIBRATE_ENV.to_string(), size.to_string()); + let mut iter_config = config.clone(); + iter_config.name = format!("{}_{}b", config.name, size); + last_metadata = run_scaling_loop( + &mut reusable_hosts, + &mut deployment, + &built, + &iter_config, + &Optimizations::default(), + &HashMap::new(), + None, + &HashSet::new(), + ) + .await; + } + last_metadata + } else { + run_scaling_loop( + &mut reusable_hosts, + &mut deployment, + &built, + &config, + &Optimizations::default(), + &HashMap::new(), + None, + &HashSet::new(), + ) + .await + }; (deep_clone(built.ir()), final_run_metadata) } diff --git a/hydro_optimize_examples/examples/benchmark_paxos.rs b/hydro_optimize_examples/examples/benchmark_paxos.rs index 36e1505..0abb432 100644 --- a/hydro_optimize_examples/examples/benchmark_paxos.rs +++ b/hydro_optimize_examples/examples/benchmark_paxos.rs @@ -120,7 +120,7 @@ async fn main() { start_virtual_clients: 1, virtual_clients_step: 10, num_runs: 1, - calibrate_message_size: None, + calibrate_message_sizes: None, }, ) }, diff --git a/hydro_optimize_examples/examples/cas.rs b/hydro_optimize_examples/examples/cas.rs index f106155..9b3077f 100644 --- a/hydro_optimize_examples/examples/cas.rs +++ b/hydro_optimize_examples/examples/cas.rs @@ -87,7 +87,7 @@ fn run_benchmark<'a>(write_ratio: usize, optimize: bool) -> (FlowBuilder<'a>, Be start_virtual_clients: 1, virtual_clients_step: VIRTUAL_CLIENTS_STEP, num_runs: 1, - calibrate_message_size: None, + calibrate_message_sizes: None, }, ) } diff --git a/hydro_optimize_examples/examples/network_calibrator.rs b/hydro_optimize_examples/examples/network_calibrator.rs index 31823ed..462145a 100644 --- a/hydro_optimize_examples/examples/network_calibrator.rs +++ b/hydro_optimize_examples/examples/network_calibrator.rs @@ -34,58 +34,54 @@ struct Args { async fn main() { let args = Args::parse(); - for &size in CALIBRATION_SIZES { - println!("=== Calibrating message_size={} ===", size); + benchmark_protocol( + BenchmarkArgs { + gcp: args.gcp.clone(), + aws: args.aws, + }, + move || { + let mut builder = FlowBuilder::new(); + let server = builder.process::(); + let clients = builder.process::(); + let client_id = clients.id(); - benchmark_protocol( - BenchmarkArgs { - gcp: args.gcp.clone(), - aws: args.aws, - }, - move || { - let mut builder = FlowBuilder::new(); - let server = builder.process::(); - let clients = builder.process::(); - let client_id = clients.id(); + let location_id_to_cluster = HashMap::from([ + (server.id(), "server".to_string()), + (client_id.clone(), "client".to_string()), + ]); - let location_id_to_cluster = HashMap::from([ - (server.id(), "server".to_string()), - (client_id.clone(), "client".to_string()), - ]); + // Dummy code. Just need to make sure that the server receives and sends. + // The actual message generation is done by the Hydro IR. + clients + .source_iter(q!(vec![0usize])) + .send(&server, TCP.fail_stop().bincode()) + .send(&clients, TCP.fail_stop().bincode()); - // Dummy code. Just need to make sure that the server receives and sends. - // The actual message generation is done by the Hydro IR. - clients - .source_iter(q!(vec![0usize])) - .send(&server, TCP.fail_stop().bincode()) - .send(&clients, TCP.fail_stop().bincode()); + let clusters = ReusableClusters::default(); + let processes = ReusableProcesses::default() + .with_process(server) + .with_process(clients); + let optimizations = Optimizations::default().excluding(client_id); - let clusters = ReusableClusters::default(); - let processes = ReusableProcesses::default() - .with_process(server) - .with_process(clients); - let optimizations = Optimizations::default().excluding(client_id); - - ( - builder, - BenchmarkConfig { - name: format!("Network_{}b", size), - workload: "default".to_string(), - clusters, - processes, - optimizations, - location_id_to_cluster, - num_physical_clients: 1, - start_virtual_clients: 1, - virtual_clients_step: 1, - num_runs: 1, - calibrate_message_size: Some(size), - }, - ) - }, - ) - .await; - } + ( + builder, + BenchmarkConfig { + name: "Network".to_string(), + workload: "default".to_string(), + clusters, + processes, + optimizations, + location_id_to_cluster, + num_physical_clients: 1, + start_virtual_clients: 1, + virtual_clients_step: 1, + num_runs: 1, + calibrate_message_sizes: Some(CALIBRATION_SIZES.to_vec()), + }, + ) + }, + ) + .await; println!("=== Calibration complete ==="); } diff --git a/setup_gurobi.sh b/setup_gurobi.sh index 520488e..fe7d0b0 100755 --- a/setup_gurobi.sh +++ b/setup_gurobi.sh @@ -6,5 +6,5 @@ sudo wget https://packages.gurobi.com/13.0/gurobi13.0.2_linux64.tar.gz sudo tar xvfz gurobi13.0.2_linux64.tar.gz echo 'export GUROBI_HOME="/opt/gurobi1302/linux64"' >> ~/.zshrc echo 'export PATH="${PATH}:${GUROBI_HOME}/bin"' >> ~/.zshrc -echo 'export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${GUROBI_HOME}/lib"' >> ~/.zshrc +echo 'export LD_LIBRARY_PATH="${GUROBI_HOME}/lib"' >> ~/.zshrc source ~/.zshrc From 9d0075485cf5c3cb13f0ef89e0c81a306a77088d Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Fri, 12 Jun 2026 22:01:51 +0000 Subject: [PATCH 03/29] Parse process results for network calibrator --- hydro_optimize/src/parse_results.rs | 55 +++++++++++++++-------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index 07e4f6a..c4cb336 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -1281,35 +1281,38 @@ pub async fn analyze_cluster_results( )); } - // Merge metrics across nodes in each cluster - for (id, name, _cluster) in nodes.get_all_clusters() { - let cluster_data = drained.get(&(id.clone(), name.to_string())).unwrap(); - - let mut max_sar: Vec = vec![]; - let mut max_counters: HashMap> = HashMap::new(); - - for (sar_stats, counters, _, _, byte_sizes) in cluster_data { - merge_max_vec(&mut max_sar, sar_stats, SarStats::max); - for (op_id, rates) in counters { - let entry = max_counters.entry(*op_id).or_default(); - merge_max_vec(entry, rates, usize::max); - } - for (op_id, sizes) in byte_sizes { - run_metadata - .byte_sizes - .entry(*op_id) - .or_default() - .extend(sizes); + // Merge metrics across nodes in each cluster and process + let all_nodes = nodes.get_all_clusters().map(|(id, name, _)| (id, name)) + .chain(nodes.get_all_processes().map(|(id, name, _)| (id, name))); + + for (id, name) in all_nodes { + if let Some(cluster_data) = drained.get(&(id.clone(), name.to_string())) { + let mut max_sar: Vec = vec![]; + let mut max_counters: HashMap> = HashMap::new(); + + for (sar_stats, counters, _, _, byte_sizes) in cluster_data { + merge_max_vec(&mut max_sar, sar_stats, SarStats::max); + for (op_id, rates) in counters { + let entry = max_counters.entry(*op_id).or_default(); + merge_max_vec(entry, rates, usize::max); + } + for (op_id, sizes) in byte_sizes { + run_metadata + .byte_sizes + .entry(*op_id) + .or_default() + .extend(sizes); + } } - } - for (op_id, rates) in max_counters { - let entry = op_to_count.entry(op_id).or_default(); - merge_max_vec(entry, &rates, usize::max); - } + for (op_id, rates) in max_counters { + let entry = op_to_count.entry(op_id).or_default(); + merge_max_vec(entry, &rates, usize::max); + } - if !max_sar.is_empty() { - run_metadata.sar_stats.insert(id.clone(), max_sar); + if !max_sar.is_empty() { + run_metadata.sar_stats.insert(id.clone(), max_sar); + } } } From 26f260262157800c4bfdf84d0e60aeea50de1bc1 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Fri, 12 Jun 2026 22:02:03 +0000 Subject: [PATCH 04/29] Deploy to us-east-1 for capacity reasons --- hydro_optimize/src/deploy.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hydro_optimize/src/deploy.rs b/hydro_optimize/src/deploy.rs index af06343..45e8640 100644 --- a/hydro_optimize/src/deploy.rs +++ b/hydro_optimize/src/deploy.rs @@ -36,8 +36,8 @@ pub struct ReusableHosts { } // Note: Aws AMIs vary by region. If you are changing the region, please also change the AMI. -const AWS_REGION: &str = "us-west-2"; -const AWS_INSTANCE_AMI: &str = "ami-055a9df0c8c9f681c"; // Amazon Linux 2 +const AWS_REGION: &str = "us-east-1"; +const AWS_INSTANCE_AMI: &str = "ami-0521cb2d60cfbb1a6"; // Amazon Linux 2023 const AWS_INSTANCE_TYPE: &str = "m5.2xlarge"; // 8 vCPU, 32 GB RAM const AWS_NUM_CORES: usize = 8; // Used for pinning /// m5.2xlarge: up to 10 Gbps network bandwidth From aed706b9cf41faae4ca399269f310acdccd6a0f5 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Fri, 12 Jun 2026 22:02:15 +0000 Subject: [PATCH 05/29] Use fewer client machines --- hydro_optimize/src/deploy_and_analyze.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index e5eaa89..4ffd6bf 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -608,7 +608,7 @@ pub struct BenchmarkConfig { pub const START_MEASUREMENT_SECOND: usize = 30; pub const MEASUREMENT_SECOND: usize = 59; pub const RUN_SECONDS: usize = 90; -pub const NUM_PHYSICAL_CLIENTS: usize = 10; +pub const NUM_PHYSICAL_CLIENTS: usize = 5; pub const VIRTUAL_CLIENTS_STEP: usize = 50; // Can tweak to get finer-grained numbers pub const NUM_VIRTUAL_CLIENTS_ENV: &str = "NUM_VIRTUAL_CLIENTS"; pub const NUM_RUNS_NO_THROUGHPUT: usize = 3; From 293d437bcec64dfe813b9a841f6e327c98a1dbf4 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Fri, 12 Jun 2026 15:46:21 -0700 Subject: [PATCH 06/29] Amazon linux install instructions --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 1ea2d32..e8c4b33 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,19 @@ Automatically apply decoupling and partitioning to Hydro programs for higher throughput. -> [!NOTE] -> Only Linux is supported, as we compile with `glibc` for better performance and more legibile `perf` results. +The following instructions assume you are running this from an Amazon Linux 2023 machine. +This is because we compile with `glibc` for better performance and more legibile `perf` results, and we need to launch from a machine with the same `glibc` version as the destination, which currently is an Amazon Linux 2023 instance. + +The launch instance types are specified in `deploy.rs`; if you wish to compile from another machine, be sure to edit the instance constants in this file. +The GCP code path is untested but should also work. ## Installation +Install development tools. +```bash +sudo dnf groupinstall -y "Development Tools" +sudo dnf install -y tmux +``` ### Rust Install Rust. Install `rust-analyzer` as well if you plan on editing the code in an IDE. @@ -17,12 +25,7 @@ rustup component add rust-analyzer ``` ### Permissions -The Linux machine on which you are running hydro-optimize will need permissions to launch VMs. If this is your local machine, you can simply sign into your AWS account locally; otherwise you will need to grant the remote machine the appropriate permissions. - -#### AWS EC2 -The launch instance types are specified in `deploy.rs`. -Since we use `glibc` for compilation, we need to launch from a machine with the same `glibc` version as the destination, which currently is an Amazon Linux 2023 instance. -Do not use another Linux machine to launch for this reason. +The machine on which you are running hydro-optimize will need permissions to launch VMs. 1. Go to AWS IAM Roles > Create role. 2. Select "EC2" as the Use case. 3. Add "AmazonEC2FullAccess" as the Permission policy. @@ -30,8 +33,7 @@ Do not use another Linux machine to launch for this reason. 5. Go to the Instance you are running hydro-optimize from, Actions > Security > Modify IAM role. 6. Give it the EC2FullRole. -#### GCP Compute Engine -TODO +You will need to increase your EC2 quota for experiments like `benchmark_paxos`. Request an increase to your "Running On-Demand Standard (A, C, D, H, I, M, R, T, Z) instances" quota for `us-east-1` to 500. ### Terraform Terraform is used to spin up machines. @@ -39,7 +41,7 @@ These are instructions for Amazon Linux machines; refer to [the official site](h ```bash sudo dnf install -y dnf-plugins-core sudo dnf config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo -sudo dnf -y install terraform +sudo dnf install -y terraform ``` ### ILP Solver @@ -48,16 +50,14 @@ Gurobi is not free; you will need to either create a Gurobi account (and use a f ```bash ./setup_gurobi.sh +source ~/.zshrc ``` Create a Gurobi license following your organization's instructions. -For free educational licenses, you may request a named license from Gurobi, then run the command they provide, which should resemble the following: - -```bash -grbgetkey -``` +For free educational licenses, you may request a license from Gurobi, then download it (which should create a gurobi.lic file) and put it on your remote machine under `~/`. ## Execution +Run all commands below within `tmux` so they continue running even if your SSH connection breaks. To run Microbus or Krupa without applying any optimizations, run: ```bash From 7fb451327f5c78c1145ca4077ddbce2d6d41e444 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Mon, 15 Jun 2026 16:48:33 +0000 Subject: [PATCH 07/29] Remove socket counts from network costs, estimated costs extremely low... --- hydro_optimize/src/decouple_analysis.rs | 68 ++++++++++- hydro_optimize/src/parse_results.rs | 156 ++++++------------------ 2 files changed, 103 insertions(+), 121 deletions(-) diff --git a/hydro_optimize/src/decouple_analysis.rs b/hydro_optimize/src/decouple_analysis.rs index d719abc..80e8fc4 100644 --- a/hydro_optimize/src/decouple_analysis.rs +++ b/hydro_optimize/src/decouple_analysis.rs @@ -64,7 +64,6 @@ pub(crate) struct DecoupleILPMetadata { pub(crate) op_sizes: HashMap, pub(crate) network_cost_table: NetworkCostTable, pub(crate) per_op_load: HashMap, - pub(crate) cluster_size: usize, pub(crate) max_num_locations: usize, // Model variables to construct final cost function pub(crate) variables: ProblemVariables, @@ -225,7 +224,6 @@ fn add_op_resource_usage( op_counts, op_sizes, network_cost_table, - cluster_size, variables, constraints, max_num_locations: num_locations, @@ -249,7 +247,7 @@ fn add_op_resource_usage( .unwrap_or(0); let size = op_sizes.get(&op_id).copied().unwrap_or(0); if count > 0 && size > 0 { - network_cost_table.network_cost(count, size, *cluster_size) + network_cost_table.network_cost(count, size) } else { return; } @@ -317,7 +315,7 @@ fn network_cost_for_decoupling_op( ) -> SarStats { let cardinality = op_counts.get(&op_id).copied().unwrap_or(0); match op_sizes.get(&op_id) { - Some(&bytes) => network_cost_table.network_cost(cardinality, bytes, 1), + Some(&bytes) => network_cost_table.network_cost(cardinality, bytes), None => SarStats::default(), } } @@ -739,7 +737,6 @@ pub(crate) fn decouple_analysis( op_sizes: op_sizes.clone(), network_cost_table: network_cost_table.clone(), per_op_load: per_op_load.clone(), - cluster_size, max_num_locations, variables: variables! {}, constraints: vec![], @@ -974,3 +971,64 @@ pub fn find_optimal_budget( results } + +/// Calculate the total CPU usage using the calibrated network cost. +/// If this exceeds 100%, then our calibration assumes network costs too much. +/// If this is under 100%, then our calibration assumes network costs too little. +/// +/// The return type is `Vec` just to match the signature of `find_optimal_budget` +pub(crate) fn project_total_cpu( + ir: &mut [HydroRoot], + bottleneck: &LocationId, + inputs: &IlpInputs, + cycle_source_to_sink_parent: &HashMap, +) -> Vec { + let op_id_to_parents = op_id_to_parents(ir, Some(bottleneck), cycle_source_to_sink_parent); + let mut total_cpu = 0f64; + + traverse_dfir( + ir, + |_, _| {}, + |node, _| { + let network_type = get_network_type(node, bottleneck); + if network_type.is_none() && node.metadata().location_id.root() != bottleneck { + return; + } + + let op_id = node.op_metadata().id.unwrap(); + + if let Some(network_type) = network_type { + let parent_id = op_id_to_parents + .get(&op_id) + .and_then(|p| p.first().copied()); + let count = inputs + .op_counts + .get(&parent_id.unwrap_or(op_id)) + .copied() + .unwrap_or(0); + let size = inputs.op_output_sizes.get(&op_id).copied().unwrap_or(0); + if count > 0 && size > 0 { + let cost = inputs + .network_cost_table + .network_cost(count, size); + println!("Network op {}: count={}, size={}, cpu_cost={:.2}", op_id, count, size, cost.cpu); + + if network_type == NetworkType::SendRecv { + total_cpu += cost.cpu * 2.0; // Pay both send & receive + } + else { + total_cpu += cost.cpu; + } + } + } else { + if let Some(load) = inputs.per_op_load.get(&op_id) { + println!("Load for op {}: cpu={}, mem={}, net={}, io={}", op_id, load.cpu, load.memory, load.network, load.io); + total_cpu += load.cpu; + } + } + }, + ); + + println!("Projected total CPU usage at bottleneck: {:.2}%", total_cpu); + std::process::exit(0); +} diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index c4cb336..d6e446f 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -234,7 +234,6 @@ pub fn load_raw_ilp_inputs(base_dir: &Path, name: &str) -> RawIlpInputs { let counters_dirs = find_workload_dirs(base_dir, name, OptimizationKind::CountersOnly.label()); let perf_dirs = find_workload_dirs(base_dir, name, OptimizationKind::PerfOnly.label()); - assert!(!blow_up_dirs.is_empty(), "No blow_up dirs for '{}'", name); assert!(!size_dirs.is_empty(), "No size dirs for '{}'", name); assert!(!counters_dirs.is_empty(), "No counters dirs for '{}'", name); assert!(!perf_dirs.is_empty(), "No perf dirs for '{}'", name); @@ -253,20 +252,6 @@ pub fn load_raw_ilp_inputs(base_dir: &Path, name: &str) -> RawIlpInputs { } } } - - let op_output_sizes = max_across_dirs(&size_dirs, |dir| { - let path = dir.join("size_analysis.json"); - if path.exists() { - load_json(&path) - } else { - HashMap::new() - } - }); - let op_counts = max_across_dirs(&counters_dirs, |dir| { - load_json_for_best_csv(dir, "counters") - }); - let perf = max_across_dirs(&perf_dirs, |dir| load_json_for_best_csv(dir, "perf")); - // Convert blow-up stats to per-op memory/IO let mut per_op_blow_up: HashMap = HashMap::new(); for stats in blow_up_stats.values() { @@ -281,6 +266,19 @@ pub fn load_raw_ilp_inputs(base_dir: &Path, name: &str) -> RawIlpInputs { } } + let op_output_sizes = max_across_dirs(&size_dirs, |dir| { + let path = dir.join("size_analysis.json"); + if path.exists() { + load_json(&path) + } else { + HashMap::new() + } + }); + let op_counts = max_across_dirs(&counters_dirs, |dir| { + load_json_for_best_csv(dir, "counters") + }); + let perf = max_across_dirs(&perf_dirs, |dir| load_json_for_best_csv(dir, "perf")); + RawIlpInputs { per_op_blow_up, op_output_sizes, @@ -294,9 +292,9 @@ pub fn load_raw_ilp_inputs(base_dir: &Path, name: &str) -> RawIlpInputs { /// Averages cost-per-message across all runs for each (message_size, num_sockets) pair. /// Clients are converted to sockets: num_sockets = num_clients * 2. pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { - let dir_regex = Regex::new(r"^Network_(\d+)b_(\d+)c_default_none$").unwrap(); - // Accumulate per (num_sockets, msg_size) -> Vec across runs - let mut per_key: HashMap<(usize, u64), Vec> = HashMap::new(); + let dir_regex = Regex::new(r"^Network_(\d+)b_default_none$").unwrap(); + // Accumulate per msg_size -> Vec across runs + let mut per_key: HashMap> = HashMap::new(); for entry in fs::read_dir(calibration_dir).expect("Failed to read calibration directory") { let entry = entry.unwrap(); @@ -306,16 +304,6 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { continue; }; let msg_size = cap[1].parse::().unwrap(); - let num_clients = cap[2].parse::().unwrap(); - // Empirically, avg active sockets per tick is ~1 less than num_clients for 4 clients. - // For 10 clients, source stats aren't recorded but behave similarly to sink. - // Encode 4 clients as 3 sockets, 10 clients as 10 sockets (sink-only measurement). - // Multiply by 2 for both directions (source + sink). - let num_sockets = match num_clients { - 4 => 3 * 2, - 10 => 10 * 2, - _ => continue, - }; let dir_path = entry.path(); let server_files: Vec<_> = fs::read_dir(&dir_path) @@ -358,7 +346,7 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { let throughput = (sum_thr / n) as usize; if throughput > 0 { per_key - .entry((num_sockets, msg_size)) + .entry(msg_size) .or_default() .push(SarStats { cpu: (sum_cpu / n) / throughput as f64, @@ -371,9 +359,9 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { } } - // Average across runs for each (num_sockets, msg_size) - let mut entries: HashMap> = HashMap::new(); - for ((num_sockets, msg_size), stats_vec) in per_key { + // Average across runs for each msg_size + let mut entries: HashMap = HashMap::new(); + for (msg_size, stats_vec) in per_key { let n = stats_vec.len() as f64; let avg = SarStats { cpu: stats_vec.iter().map(|s| s.cpu).sum::() / n, @@ -382,10 +370,7 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { memory: 0.0, io: 0.0, }; - entries - .entry(num_sockets) - .or_default() - .push((msg_size, avg)); + entries.insert(msg_size, avg); } assert!( @@ -393,11 +378,10 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { "No calibration data found in {:?}", calibration_dir ); - let total: usize = entries.values().map(|v| v.len()).sum(); + let total = entries.len(); println!( - "Loaded {} calibration points across {} socket counts", - total, - entries.len() + "Loaded {} calibration points", + total ); NetworkCostTable::from_calibration(entries) } @@ -549,88 +533,36 @@ pub fn get_csvs_in_dir(dir: &Path) -> Vec { .collect() } -/// Lookup table mapping (message_size, num_sockets) → per-message resource costs. -/// Stores a precomputed linear model per message size: cost = slope * num_sockets + intercept. +/// Lookup table mapping message_size → per-message resource costs. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NetworkCostTable { /// Sorted by message_size. entries: Vec, } -/// Linear model for a single message size: cost = slope * num_sockets + intercept. +/// Calibration cost for a single message size. #[derive(Debug, Clone, Serialize, Deserialize)] struct NetworkCostEntry { message_size: u64, - slope: SarStats, - intercept: SarStats, + cost: SarStats, } impl NetworkCostTable { - pub fn from_calibration(raw: HashMap>) -> Self { - let mut tables = raw; - for table in tables.values_mut() { - table.sort_by_key(|(size, _)| *size); - for (_, stats) in table.iter_mut() { - *stats = stats.scale(0.5); - } - } - - let mut keys: Vec = tables.keys().copied().collect(); - keys.sort(); - - assert!( - keys.len() >= 2, - "At least 2 physical client configurations required for network cost interpolation" - ); - - let lo_k = keys[0]; - let hi_k = *keys.last().unwrap(); - let lo_table = &tables[&lo_k]; - let hi_table = &tables[&hi_k]; - let denom = (hi_k - lo_k) as f64; - // Only use message sizes present in both tables - let hi_sizes: HashSet = hi_table.iter().map(|(s, _)| *s).collect(); - let entries: Vec = lo_table - .iter() - .filter(|(msg_size, _)| hi_sizes.contains(msg_size)) - .map(|&(msg_size, cost_lo)| { - let cost_hi = hi_table.iter().find(|(s, _)| *s == msg_size).unwrap().1; - let slope = SarStats { - cpu: (cost_hi.cpu - cost_lo.cpu) / denom, - cpu_user: (cost_hi.cpu_user - cost_lo.cpu_user) / denom, - network: 0.0, - memory: 0.0, - io: 0.0, - }; - let intercept = SarStats { - cpu: cost_lo.cpu - slope.cpu * lo_k as f64, - cpu_user: cost_lo.cpu_user - slope.cpu_user * lo_k as f64, - network: cost_lo.network, // Note: This is not scaled. Number of physical clients should have no effect - memory: 0.0, - io: 0.0, - }; - NetworkCostEntry { - message_size: msg_size, - slope, - intercept, - } + pub fn from_calibration(raw: HashMap) -> Self { + let mut entries: Vec = raw + .into_iter() + .map(|(message_size, cost)| NetworkCostEntry { + message_size, + cost: cost.scale(0.5), }) .collect(); - for entry in &entries { - assert!( - entry.slope.cpu >= 0.0 && entry.slope.cpu_user >= 0.0, - "Negative slope for message_size={}: {:?}", - entry.message_size, - entry.slope - ); - } - + entries.sort_by_key(|e| e.message_size); Self { entries } } - /// Returns the cost per message for the given message size and socket count. - pub fn cost_per_message(&self, message_size_bytes: u64, num_sockets: usize) -> SarStats { + /// Returns the cost per message for the given message size + fn cost_per_message(&self, message_size_bytes: u64) -> SarStats { let n = self.entries.len(); if n == 0 { return SarStats::default(); @@ -643,14 +575,7 @@ impl NetworkCostTable { .partition_point(|e| e.message_size <= message_size_bytes); &self.entries[if i >= n { n - 1 } else { i }] }; - let x = num_sockets.max(1) as f64; - SarStats { - cpu: (entry.slope.cpu * x + entry.intercept.cpu).max(0.0), - cpu_user: (entry.slope.cpu_user * x + entry.intercept.cpu_user).max(0.0), - network: (entry.slope.network * x + entry.intercept.network).max(0.0), - memory: 0.0, - io: 0.0, - } + entry.cost } /// Total resource cost for sending `count` messages of `message_size_bytes` each. @@ -658,9 +583,8 @@ impl NetworkCostTable { &self, count: usize, message_size_bytes: u64, - num_sockets: usize, ) -> SarStats { - self.cost_per_message(message_size_bytes, num_sockets) + self.cost_per_message(message_size_bytes) .scale(count as f64) } } @@ -973,9 +897,9 @@ fn avg_over(slice: &[T], f: impl Fn(&T) -> f64) -> f64 { /// Per-second SAR statistics #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct SarStats { - /// Max single-core CPU usage (user + system %) + /// Max single-core CPU usage (user + system %), out of 100 pub cpu: f64, - /// Max single-core CPU user % only (excludes system) + /// Max single-core CPU user % only (excludes system), out of 100 pub cpu_user: f64, /// Network in + out bytes/sec pub network: f64, From 6485c3414b148ddd29d5f01bb0a8e4cda73999bd Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 16 Jun 2026 19:29:47 +0000 Subject: [PATCH 08/29] Use musl instead of glibc for compilation, parsing shows process sarstats as well --- .cargo/config.toml | 24 +++++++++++++ README.md | 10 ++---- hydro_optimize/src/deploy.rs | 11 +++--- hydro_optimize/src/parse_results.rs | 52 +++++++++++++++-------------- 4 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..8b5b2bf --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,24 @@ +[build] + +[env] +# Stack backtraces. +RUST_BACKTRACE = "1" +# Set output levels for `tracing` logging. +# Certain wasm crates emit a lot of `debug` logs, set to `info`. +RUST_LOG = "info,hydro_lang=debug,dfir_rs=trace,walrus=info,wasm_bindgen_cli_support=info,wasm_bindgen_wasm_interpreter=info" +# DFIR generate for `/docs/` website. +DFIR_GENERATE_DOCS = "1" +DFIR_BASE_DIR = { value = ".", relative = true } + +[target.aarch64-apple-darwin] +linker = "rust-lld" + +[target.x86_64-apple-darwin] +linker = "rust-lld" + +[target.x86_64-unknown-linux-musl] +linker = "rust-lld" + + +[target.x86_64-pc-windows-msvc] +linker = "rust-lld.exe" diff --git a/README.md b/README.md index 1ea2d32..bdd71a8 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,6 @@ Automatically apply decoupling and partitioning to Hydro programs for higher throughput. -> [!NOTE] -> Only Linux is supported, as we compile with `glibc` for better performance and more legibile `perf` results. - - ## Installation ### Rust @@ -13,6 +9,9 @@ Install Rust. Install `rust-analyzer` as well if you plan on editing the code in ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh source ~/.bashrc +rustup target add x86_64-unknown-linux-musl + +# Optional rustup component add rust-analyzer ``` @@ -20,9 +19,6 @@ rustup component add rust-analyzer The Linux machine on which you are running hydro-optimize will need permissions to launch VMs. If this is your local machine, you can simply sign into your AWS account locally; otherwise you will need to grant the remote machine the appropriate permissions. #### AWS EC2 -The launch instance types are specified in `deploy.rs`. -Since we use `glibc` for compilation, we need to launch from a machine with the same `glibc` version as the destination, which currently is an Amazon Linux 2023 instance. -Do not use another Linux machine to launch for this reason. 1. Go to AWS IAM Roles > Create role. 2. Select "EC2" as the Use case. 3. Add "AmazonEC2FullAccess" as the Permission policy. diff --git a/hydro_optimize/src/deploy.rs b/hydro_optimize/src/deploy.rs index 45e8640..3e1603d 100644 --- a/hydro_optimize/src/deploy.rs +++ b/hydro_optimize/src/deploy.rs @@ -8,6 +8,8 @@ use hydro_deploy::rust_crate::tracing_options::{ use hydro_deploy::{AwsNetwork, Deployment, Host, HostTargetType, LinuxCompileType}; use hydro_lang::deploy::TrybuildHost; +const COMPILE_TYPE: LinuxCompileType = LinuxCompileType::Glibc; + /// What the user provides when creating ReusableHosts #[derive(PartialEq, Eq, Clone)] pub enum HostType { @@ -113,8 +115,6 @@ impl ReusableHosts { .region(GCP_REGION) .network(network.clone()) .display_name(display_name) - // Better performance than MUSL, perf reporting fewer unidentified stacks, but requires launching from Linux - .target_type(HostTargetType::Linux(LinuxCompileType::Glibc)) .add(), InitializedHostType::Aws { network } => deployment .AwsEc2Host() @@ -123,8 +123,6 @@ impl ReusableHosts { .ami(AWS_INSTANCE_AMI) .network(network.clone()) .display_name(display_name) - // Better performance than MUSL, perf reporting fewer unidentified stacks, but requires launching from Linux - .target_type(HostTargetType::Linux(LinuxCompileType::Glibc)) .add(), InitializedHostType::Localhost => deployment.Localhost(), }) @@ -134,13 +132,12 @@ impl ReusableHosts { fn get_rust_flags(&self) -> String { match &self.host_type { InitializedHostType::Gcp { .. } | InitializedHostType::Aws { .. } => { - "-C opt-level=3 -C codegen-units=1 -C strip=none -C debuginfo=2 -C lto=off" + "-C opt-level=3 -C codegen-units=1 -C strip=none -C debuginfo=2 -C lto=off -C link-arg=--no-rosegment".to_string() } InitializedHostType::Localhost => { - "" // Compile fast! Localhost is used for debugging + "".to_string() // Compile fast! Localhost is used for debugging } } - .to_string() } fn host_with_env(&self, host: TrybuildHost) -> TrybuildHost { diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index d6e446f..c1529eb 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -345,16 +345,13 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { let throughput = (sum_thr / n) as usize; if throughput > 0 { - per_key - .entry(msg_size) - .or_default() - .push(SarStats { - cpu: (sum_cpu / n) / throughput as f64, - cpu_user: (sum_cpu_user / n) / throughput as f64, - network: (sum_net / n) / throughput as f64, - memory: 0.0, - io: 0.0, - }); + per_key.entry(msg_size).or_default().push(SarStats { + cpu: (sum_cpu / n) / throughput as f64, + cpu_user: (sum_cpu_user / n) / throughput as f64, + network: (sum_net / n) / throughput as f64, + memory: 0.0, + io: 0.0, + }); } } } @@ -379,10 +376,7 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { calibration_dir ); let total = entries.len(); - println!( - "Loaded {} calibration points", - total - ); + println!("Loaded {} calibration points", total); NetworkCostTable::from_calibration(entries) } @@ -579,11 +573,7 @@ impl NetworkCostTable { } /// Total resource cost for sending `count` messages of `message_size_bytes` each. - pub fn network_cost( - &self, - count: usize, - message_size_bytes: u64, - ) -> SarStats { + pub fn network_cost(&self, count: usize, message_size_bytes: u64) -> SarStats { self.cost_per_message(message_size_bytes) .scale(count as f64) } @@ -635,7 +625,7 @@ pub struct RunMetadata { pub sar_stats: HashMap>, /// Maps each LocationId → original op_ids assigned to it (populated when size_analysis is used) pub location_to_original_ops: HashMap>, - /// Per-operator CPU usage fractions from perf: op_id → fraction of total samples. + /// Per-operator CPU usage fractions from perf: op_id → fraction of total samples (max 1.0). pub perf: HashMap, } @@ -1116,10 +1106,20 @@ pub fn parse_perf(folded: &str) -> HashMap { let mut samples_per_id: HashMap = HashMap::new(); for line in folded.lines() { - let n_samples: f64 = line - .rsplit_once(' ') - .and_then(|(_, s)| s.parse().ok()) - .unwrap_or(0.0); + let (stack_trace, n_samples) = match line.rsplit_once(' ') { + Some((stack, count_str)) => { + let n: f64 = count_str.parse().unwrap_or(0.0); + (stack, n) + } + None => (line, 0.0), + }; + + // Skip perf numbers from IO threads + let parent = stack_trace.split(';').next().unwrap_or(stack_trace); + if parent.starts_with("hydro-io-") { + continue; + } + total_samples += n_samples; if let Some(cap) = operator_regex.captures_iter(line).last() { @@ -1206,7 +1206,9 @@ pub async fn analyze_cluster_results( } // Merge metrics across nodes in each cluster and process - let all_nodes = nodes.get_all_clusters().map(|(id, name, _)| (id, name)) + let all_nodes = nodes + .get_all_clusters() + .map(|(id, name, _)| (id, name)) .chain(nodes.get_all_processes().map(|(id, name, _)| (id, name))); for (id, name) in all_nodes { From ca893a10da4129ea8b6ad754399d720d26a92b1b Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Wed, 17 Jun 2026 18:04:50 +0000 Subject: [PATCH 09/29] Limit networking cores for performance, apply batch limit to regular deployment, sar only tracks core 0 --- hydro_optimize/src/deploy.rs | 7 ++--- hydro_optimize/src/deploy_and_analyze.rs | 18 +++++++----- hydro_optimize/src/parse_results.rs | 36 ++++++++++-------------- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/hydro_optimize/src/deploy.rs b/hydro_optimize/src/deploy.rs index 3e1603d..8c5d480 100644 --- a/hydro_optimize/src/deploy.rs +++ b/hydro_optimize/src/deploy.rs @@ -5,11 +5,9 @@ use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::rust_crate::tracing_options::{ AL2_PERF_SETUP_COMMAND, DEBIAN_PERF_SETUP_COMMAND, TracingOptions, }; -use hydro_deploy::{AwsNetwork, Deployment, Host, HostTargetType, LinuxCompileType}; +use hydro_deploy::{AwsNetwork, Deployment, Host}; use hydro_lang::deploy::TrybuildHost; -const COMPILE_TYPE: LinuxCompileType = LinuxCompileType::Glibc; - /// What the user provides when creating ReusableHosts #[derive(PartialEq, Eq, Clone)] pub enum HostType { @@ -41,7 +39,7 @@ pub struct ReusableHosts { const AWS_REGION: &str = "us-east-1"; const AWS_INSTANCE_AMI: &str = "ami-0521cb2d60cfbb1a6"; // Amazon Linux 2023 const AWS_INSTANCE_TYPE: &str = "m5.2xlarge"; // 8 vCPU, 32 GB RAM -const AWS_NUM_CORES: usize = 8; // Used for pinning +const AWS_NUM_CORES: usize = 3; // Used for networking. Network cores will be pinned to these cores - 1. Empirically tested on m5.2xlarge. /// m5.2xlarge: up to 10 Gbps network bandwidth pub const AWS_NETWORK_BYTES_PER_SEC: f64 = 1_250_000_000.0; /// m5.2xlarge: 12,000 baseline IOPS (EBS) @@ -163,7 +161,6 @@ impl ReusableHosts { }; host = host.tracing( TracingOptions::builder() - .perf_raw_outfile(format!("{}.perf.data", display_name)) .fold_outfile(format!("{}.data.folded", display_name)) .frequency(128) .setup_command(setup_command) diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index 4ffd6bf..04d9e60 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -14,7 +14,7 @@ use hydro_lang::prelude::{Cluster, FlowBuilder, Process}; use hydro_lang::telemetry::Sidecar; use tokio::sync::mpsc::UnboundedReceiver; -use crate::decouple_analysis::{IlpInputs, Rewrite, find_optimal_budget}; +use crate::decouple_analysis::{IlpInputs, Rewrite, find_optimal_budget, project_total_cpu}; use crate::deploy::{AWS_IO_TPS, AWS_NETWORK_BYTES_PER_SEC, HostType, ReusableHosts}; use crate::greedy_decouple_analysis::greedy_decouple_analysis; use crate::parse_results::{ @@ -523,9 +523,11 @@ async fn deploy_and_analyze<'a>( optimizations: &Optimizations, num_seconds: Option, ) -> RunMetadata { - // Measure network (-n DEV) and CPU (-u) usage. -P ALL measures all CPUs on the machine + // Measure network (-n DEV) and CPU (-u) usage. -P 0 measures only core 0, where the + // hydro main thread is pinned (networking threads spin on other cores, so whole-machine + // CPU is misleading). let sar_sidecar = ScriptSidecar { - script: "sar -n DEV -u -P ALL -r -b 1".to_string(), + script: "sar -n DEV -u -P 0 -r -b 1".to_string(), prefix: SAR_USAGE_PREFIX.to_string(), }; @@ -608,7 +610,7 @@ pub struct BenchmarkConfig { pub const START_MEASUREMENT_SECOND: usize = 30; pub const MEASUREMENT_SECOND: usize = 59; pub const RUN_SECONDS: usize = 90; -pub const NUM_PHYSICAL_CLIENTS: usize = 5; +pub const NUM_PHYSICAL_CLIENTS: usize = 10; pub const VIRTUAL_CLIENTS_STEP: usize = 50; // Can tweak to get finer-grained numbers pub const NUM_VIRTUAL_CLIENTS_ENV: &str = "NUM_VIRTUAL_CLIENTS"; pub const NUM_RUNS_NO_THROUGHPUT: usize = 3; @@ -1073,7 +1075,8 @@ async fn run_bottleneck_elimination<'a>( OptimizationKind::PerfOnly, OptimizationKind::CountersOnly, OptimizationKind::SizeAnalysis, - OptimizationKind::BlowUpAnalysis, + // OptimizationKind::BlowUpAnalysis, + // TODO: Reenable blow up analysis once vcpu limit has been addressed ] { if find_workload_dirs(base, &analysis_name, kind.label()).is_empty() { let run_built = analysis_built.as_ref().unwrap_or(&built); @@ -1123,7 +1126,8 @@ async fn run_bottleneck_elimination<'a>( cluster_size, }; - let computed_rewrites = find_optimal_budget(&mut ir, &bottleneck_loc, &ilp_inputs, &cycles); + // TODO: Run optimization again once projected CPU is close to 100% + let computed_rewrites = project_total_cpu(&mut ir, &bottleneck_loc, &ilp_inputs, &cycles); state .cluster_rewrites .insert(bottleneck_cluster.clone(), computed_rewrites); @@ -1215,7 +1219,7 @@ pub async fn benchmark_protocol<'a>( &mut deployment, &built, &config, - &Optimizations::default(), + &config.optimizations, &HashMap::new(), None, &HashSet::new(), diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index c1529eb..9c5d894 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -887,9 +887,9 @@ fn avg_over(slice: &[T], f: impl Fn(&T) -> f64) -> f64 { /// Per-second SAR statistics #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct SarStats { - /// Max single-core CPU usage (user + system %), out of 100 + /// Core 0 CPU usage (user + system %), out of 100 — where the hydro main thread is pinned pub cpu: f64, - /// Max single-core CPU user % only (excludes system), out of 100 + /// Core 0 CPU user % only (excludes system), out of 100 pub cpu_user: f64, /// Network in + out bytes/sec pub network: f64, @@ -930,10 +930,10 @@ impl SarStats { } } -/// Parses `sar -n DEV -u -P ALL -r -b` output lines and returns per-second SarStats. +/// Parses `sar -n DEV -u -P 0 -r -b` output lines and returns per-second SarStats. pub fn parse_sar_output(lines: Vec) -> Vec { let cpu_regex = Regex::new( - r"\s(all|\d{1,3})\s+(\d+\.?\d*)\s+\d+\.?\d*\s+(\d+\.?\d*)\s+\d+\.?\d*\s+\d+\.?\d*\s+(\d+\.?\d*)$", + r"\s(\d{1,3})\s+(\d+\.?\d*)\s+\d+\.?\d*\s+(\d+\.?\d*)\s+\d+\.?\d*\s+\d+\.?\d*\s+(\d+\.?\d*)$", ).unwrap(); let iface_regex = Regex::new(r"([a-zA-Z]\S*)\s+(\d+\.?\d*)\s+(\d+\.?\d*)\s+(\d+\.?\d*)\s+(\d+\.?\d*)") @@ -952,25 +952,19 @@ pub fn parse_sar_output(lines: Vec) -> Vec { let mut result: Vec = vec![]; for line in &lines { - // CPU: "all" marks a new second, per-core lines update the max + // CPU: we run `sar -P 0`, so the core-0 line (the only CPU line) marks each new + // second. We track core 0 only, where the hydro main thread is pinned; networking + // threads spin on other cores, so whole-machine CPU is no longer meaningful. if let Some(caps) = cpu_regex.captures(line) { - let is_all = &caps[1] == "all"; - if let (Some(user), Some(system)) = - (caps[2].parse::().ok(), caps[3].parse::().ok()) + if &caps[1] == "0" + && let (Some(user), Some(system)) = + (caps[2].parse::().ok(), caps[3].parse::().ok()) { - let usage = user + system; - if is_all { - result.push(SarStats { - cpu: usage, - cpu_user: user, - ..Default::default() - }); - } else if let Some(last) = result.last_mut() - && usage > last.cpu - { - last.cpu = usage; - last.cpu_user = user; - } + result.push(SarStats { + cpu: user + system, + cpu_user: user, + ..Default::default() + }); } continue; } From 101ae0c7c9a2b7e0d29c14544126430e9abd3005 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Sat, 20 Jun 2026 05:47:14 +0000 Subject: [PATCH 10/29] Decouple analysis correctly accounts for the cardinality of broadcasts --- hydro_optimize/src/decouple_analysis.rs | 176 +++++++++--------- hydro_optimize/src/deploy_and_analyze.rs | 10 +- hydro_optimize/src/parse_results.rs | 9 +- .../examples/network_calibrator.rs | 16 +- 4 files changed, 112 insertions(+), 99 deletions(-) diff --git a/hydro_optimize/src/decouple_analysis.rs b/hydro_optimize/src/decouple_analysis.rs index 80e8fc4..f183f67 100644 --- a/hydro_optimize/src/decouple_analysis.rs +++ b/hydro_optimize/src/decouple_analysis.rs @@ -19,9 +19,7 @@ use good_lp::{ }; use hydro_lang::compile::builder::ClockId; -use hydro_lang::compile::ir::{ - HydroIrMetadata, HydroIrOpMetadata, HydroNode, HydroRoot, traverse_dfir, -}; +use hydro_lang::compile::ir::{HydroIrMetadata, HydroNode, HydroRoot, traverse_dfir}; use hydro_lang::location::dynamic::LocationId; use super::rewrites::{NetworkType, get_network_type}; @@ -213,8 +211,27 @@ fn add_tick_constraint( } } +/// Adds `load * var` (per resource) at each variable's location. +fn add_load_to_locations( + resource_usages: &mut HashMap, + vars: &HashMap, + load: &SarStats, +) { + for (loc, var) in vars { + let res = resource_usages.get_mut(loc).unwrap(); + let cpu_temp = std::mem::take(&mut res.cpu); + res.cpu = cpu_temp + load.cpu * *var; + let mem_temp = std::mem::take(&mut res.memory); + res.memory = mem_temp + load.memory * *var; + let net_temp = std::mem::take(&mut res.network); + res.network = net_temp + load.network * *var; + let io_temp = std::mem::take(&mut res.io); + res.io = io_temp + load.io * *var; + } +} + fn add_op_resource_usage( - metadata: &HydroIrOpMetadata, + node: &HydroNode, network_type: &Option, op_id_to_parents: &HashMap>, decoupling_metadata: &RefCell, @@ -232,75 +249,53 @@ fn add_op_resource_usage( .. } = &mut *decoupling_metadata.borrow_mut(); - let op_id = metadata.id.unwrap(); - let op_load = if let Some(load) = per_op_load.get(&op_id) { - *load - } else if network_type.is_some() { - // Network ops aren't in per_op_load; derive cost from calibration table - // Use parent's count (total messages sent) and the network op's own size - let parent_id = op_id_to_parents - .get(&op_id) - .and_then(|p| p.first().copied()); - let count = op_counts - .get(&parent_id.unwrap_or(op_id)) - .copied() - .unwrap_or(0); - let size = op_sizes.get(&op_id).copied().unwrap_or(0); - if count > 0 && size > 0 { - network_cost_table.network_cost(count, size) - } else { - return; - } - } else { - return; - }; - if op_load.cpu <= 0.0 && op_load.memory <= 0.0 && op_load.network <= 0.0 && op_load.io <= 0.0 { - return; - } + let op_id = node.op_metadata().id.unwrap(); + let size = op_sizes.get(&op_id).copied().unwrap_or(0); - // Operators are run on the machine that their parents send to. + // Send / produce side: charged at the location its parent sends to. Non-network + // ops also land here — their measured compute load is paid where their parent runs. match network_type { Some(NetworkType::Send) | Some(NetworkType::SendRecv) | None => { - if let Some(parents) = op_id_to_parents.get(&op_id) - && let Some(first_parent) = parents.first() + let (load, charge_at) = if network_type.is_some() { + // Send cardinality = cardinality of Network's parent. Will be larger than the Network's cardinality if it is a broadcast + let input_id = node.input_metadata()[0].op.id; + let send_count = input_id + .and_then(|iid| op_counts.get(&iid).copied()) + .unwrap_or(0); + (network_cost_table.network_cost(send_count, size), input_id) + } else { + // Non-network op: its measured per-op load, charged at its parent. + let load = per_op_load.get(&op_id).copied().unwrap_or_default(); + let parent = op_id_to_parents + .get(&op_id) + .and_then(|p| p.first().copied()); + (load, parent) + }; + if !load.is_zero() + && let Some(charge_at) = charge_at { - let parent_vars = var_from_op_id( - *first_parent, + let vars = var_from_op_id( + charge_at, *num_locations, op_id_to_var, variables, constraints, ); - for (loc, parent_var) in parent_vars { - let res = resource_usages.get_mut(&loc).unwrap(); - let cpu_temp = std::mem::take(&mut res.cpu); - res.cpu = cpu_temp + op_load.cpu * parent_var; - let mem_temp = std::mem::take(&mut res.memory); - res.memory = mem_temp + op_load.memory * parent_var; - let net_temp = std::mem::take(&mut res.network); - res.network = net_temp + op_load.network * parent_var; - let io_temp = std::mem::take(&mut res.io); - res.io = io_temp + op_load.io * parent_var; - } + add_load_to_locations(resource_usages, &vars, &load); } } _ => {} } - // Special case for network receives: their cost (deserialization) is paid by the receiver. + // Receive side: deserialization is paid by the receiver (this op's location). The + // count is singular — the receiver's own counter already reflects the fan-in. match network_type { Some(NetworkType::Recv) | Some(NetworkType::SendRecv) => { - let op_vars = - var_from_op_id(op_id, *num_locations, op_id_to_var, variables, constraints); - for (loc, parent_var) in op_vars { - let res = resource_usages.get_mut(&loc).unwrap(); - let cpu_temp = std::mem::take(&mut res.cpu); - res.cpu = cpu_temp + op_load.cpu * parent_var; - let mem_temp = std::mem::take(&mut res.memory); - res.memory = mem_temp + op_load.memory * parent_var; - let net_temp = std::mem::take(&mut res.network); - res.network = net_temp + op_load.network * parent_var; - let io_temp = std::mem::take(&mut res.io); - res.io = io_temp + op_load.io * parent_var; + let load = + network_cost_table.network_cost(op_counts.get(&op_id).copied().unwrap_or(0), size); + if !load.is_zero() { + let vars = + var_from_op_id(op_id, *num_locations, op_id_to_var, variables, constraints); + add_load_to_locations(resource_usages, &vars, &load); } } _ => {} @@ -565,12 +560,7 @@ fn decouple_analysis_node( add_decoupling_overhead(node, op_id_to_parents, decoupling_metadata); } - add_op_resource_usage( - node.op_metadata(), - &network_type, - op_id_to_parents, - decoupling_metadata, - ); + add_op_resource_usage(node, &network_type, op_id_to_parents, decoupling_metadata); add_tick_constraint(node.metadata(), op_id_to_parents, decoupling_metadata); } @@ -981,9 +971,8 @@ pub(crate) fn project_total_cpu( ir: &mut [HydroRoot], bottleneck: &LocationId, inputs: &IlpInputs, - cycle_source_to_sink_parent: &HashMap, + _cycle_source_to_sink_parent: &HashMap, ) -> Vec { - let op_id_to_parents = op_id_to_parents(ir, Some(bottleneck), cycle_source_to_sink_parent); let mut total_cpu = 0f64; traverse_dfir( @@ -998,31 +987,46 @@ pub(crate) fn project_total_cpu( let op_id = node.op_metadata().id.unwrap(); if let Some(network_type) = network_type { - let parent_id = op_id_to_parents - .get(&op_id) - .and_then(|p| p.first().copied()); - let count = inputs - .op_counts - .get(&parent_id.unwrap_or(op_id)) - .copied() - .unwrap_or(0); let size = inputs.op_output_sizes.get(&op_id).copied().unwrap_or(0); - if count > 0 && size > 0 { - let cost = inputs - .network_cost_table - .network_cost(count, size); - println!("Network op {}: count={}, size={}, cpu_cost={:.2}", op_id, count, size, cost.cpu); - - if network_type == NetworkType::SendRecv { - total_cpu += cost.cpu * 2.0; // Pay both send & receive - } - else { - total_cpu += cost.cpu; + let net_cpu = |count: usize| { + if count > 0 && size > 0 { + inputs.network_cost_table.network_cost(count, size).cpu + } else { + 0.0 } + }; + + // Send cardinality = cardinality of Network's parent. Will be larger than the Network's cardinality if it is a broadcast + let send_count = + if matches!(network_type, NetworkType::Send | NetworkType::SendRecv) { + let input_id = node.input_metadata()[0].op.id.unwrap(); + inputs.op_counts.get(&input_id).copied().unwrap_or(0) + } else { + 0 + }; + + // Receive cardinality = cardinality of Network + let recv_count = + if matches!(network_type, NetworkType::Recv | NetworkType::SendRecv) { + inputs.op_counts.get(&op_id).copied().unwrap_or(0) + } else { + 0 + }; + + let cpu_cost = net_cpu(send_count) + net_cpu(recv_count); + if cpu_cost > 0.0 { + println!( + "Network op {}: send_count={}, recv_count={}, size={}, cpu_cost={:.2}", + op_id, send_count, recv_count, size, cpu_cost + ); + total_cpu += cpu_cost; } } else { if let Some(load) = inputs.per_op_load.get(&op_id) { - println!("Load for op {}: cpu={}, mem={}, net={}, io={}", op_id, load.cpu, load.memory, load.network, load.io); + println!( + "Load for op {}: cpu={}, mem={}, net={}, io={}", + op_id, load.cpu, load.memory, load.network, load.io + ); total_cpu += load.cpu; } } diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index 04d9e60..1062b44 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -31,7 +31,7 @@ use crate::rewrites::{ }; const METRIC_INTERVAL_SECS: u64 = 1; -const BATCH_PULL_LIMIT: usize = 1; +const BATCH_PULL_LIMIT: usize = 10; const CALIBRATION_DIR: &str = "benchmark_results"; const COUNTER_PREFIX: &str = "_optimize_counter"; const BYTE_SIZE_PREFIX: &str = "_optimize_byte_size"; @@ -611,7 +611,6 @@ pub const START_MEASUREMENT_SECOND: usize = 30; pub const MEASUREMENT_SECOND: usize = 59; pub const RUN_SECONDS: usize = 90; pub const NUM_PHYSICAL_CLIENTS: usize = 10; -pub const VIRTUAL_CLIENTS_STEP: usize = 50; // Can tweak to get finer-grained numbers pub const NUM_VIRTUAL_CLIENTS_ENV: &str = "NUM_VIRTUAL_CLIENTS"; pub const NUM_RUNS_NO_THROUGHPUT: usize = 3; pub const NO_IMPROVEMENT_LIMIT: usize = 3; @@ -1190,7 +1189,10 @@ pub async fn benchmark_protocol<'a>( run_bottleneck_elimination(&mut reusable_hosts, &mut deployment, built, &mut config) .await } - _ => panic!("Explicit optimization kinds are no longer allowed as parameters"), + _ => { + println!("Warning: Explicit optimization kinds are no longer allowed as parameters"); + built + } }; // Run a loop for each network calibration size, if provided @@ -1205,7 +1207,7 @@ pub async fn benchmark_protocol<'a>( &mut deployment, &built, &iter_config, - &Optimizations::default(), + &config.optimizations, &HashMap::new(), None, &HashSet::new(), diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index 9c5d894..d3d2310 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -558,7 +558,7 @@ impl NetworkCostTable { /// Returns the cost per message for the given message size fn cost_per_message(&self, message_size_bytes: u64) -> SarStats { let n = self.entries.len(); - if n == 0 { + if n == 0 || message_size_bytes == 0 { return SarStats::default(); } let entry = if n == 1 || message_size_bytes >= self.entries[n - 1].message_size { @@ -574,6 +574,9 @@ impl NetworkCostTable { /// Total resource cost for sending `count` messages of `message_size_bytes` each. pub fn network_cost(&self, count: usize, message_size_bytes: u64) -> SarStats { + if count == 0 || message_size_bytes == 0 { + return SarStats::default() + } self.cost_per_message(message_size_bytes) .scale(count as f64) } @@ -928,6 +931,10 @@ impl SarStats { self.memory += other.memory; self.io += other.io; } + + pub fn is_zero(&self) -> bool { + self.cpu == 0.0 && self.cpu_user == 0.0 && self.network == 0.0 && self.memory == 0.0 && self.io == 0.0 + } } /// Parses `sar -n DEV -u -P 0 -r -b` output lines and returns per-second SarStats. diff --git a/hydro_optimize_examples/examples/network_calibrator.rs b/hydro_optimize_examples/examples/network_calibrator.rs index 462145a..4321b25 100644 --- a/hydro_optimize_examples/examples/network_calibrator.rs +++ b/hydro_optimize_examples/examples/network_calibrator.rs @@ -42,7 +42,7 @@ async fn main() { move || { let mut builder = FlowBuilder::new(); let server = builder.process::(); - let clients = builder.process::(); + let clients = builder.cluster::(); let client_id = clients.id(); let location_id_to_cluster = HashMap::from([ @@ -52,16 +52,16 @@ async fn main() { // Dummy code. Just need to make sure that the server receives and sends. // The actual message generation is done by the Hydro IR. + // Use a cluster of clients (not process) since that has higher overhead clients - .source_iter(q!(vec![0usize])) + .source_iter(q!(vec![Vec::::new()])) .send(&server, TCP.fail_stop().bincode()) - .send(&clients, TCP.fail_stop().bincode()); + .demux(&clients, TCP.fail_stop().bincode()); - let clusters = ReusableClusters::default(); - let processes = ReusableProcesses::default() - .with_process(server) - .with_process(clients); - let optimizations = Optimizations::default().excluding(client_id); + let clusters = ReusableClusters::default().with_cluster(clients, 1); + let processes = ReusableProcesses::default().with_process(server); + let optimizations = Optimizations::default() + .excluding(client_id); ( builder, From 17341a57ae8b62a580f14c2ed6c9640c509b8b76 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 23 Jun 2026 20:40:50 +0000 Subject: [PATCH 11/29] Partition map insertion fix --- hydro_optimize/src/rewriter.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hydro_optimize/src/rewriter.rs b/hydro_optimize/src/rewriter.rs index 087746b..979fa53 100644 --- a/hydro_optimize/src/rewriter.rs +++ b/hydro_optimize/src/rewriter.rs @@ -69,6 +69,9 @@ struct NetworkMetadata { } /// Creates the Map before Network to route it to the correct partition. +/// A map is appended to the node passed in. +/// If a network already exists, pass in its input. +/// If a network is being created, pass node into Network after calling this function. fn map_before_network(node: &mut HydroNode, network_metadata: &NetworkMetadata) { let metadata = node.metadata().clone(); let node_content = std::mem::replace(node, HydroNode::Placeholder); @@ -485,7 +488,7 @@ fn repair_existing_network_for_partitioning( // If the receiver is now partitioned, add a Map before the Network to route to the correct partition if receiver_partitions > 0 { - map_before_network(node, &network_metadata); + map_before_network(input.as_mut(), &network_metadata); } // If the source is now partitioned, departition the ID on the receiving end if sender_partitions > 0 { From 6ca560c32f4ddfb4b422a159459149db000e330b Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 23 Jun 2026 20:41:19 +0000 Subject: [PATCH 12/29] Optimization folder overwriting fix, limit number of unstable throughput runs --- hydro_optimize/src/deploy_and_analyze.rs | 43 +++++++++++++----------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index 1062b44..fb1ee9e 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -420,12 +420,12 @@ pub enum OptimizationKind { impl OptimizationKind { pub fn label(&self) -> &'static str { match self { - OptimizationKind::None => "none", + // Do not differentiate between None and BottleneckElimination, since both run the protocol without additional profiling + OptimizationKind::None | OptimizationKind::BottleneckElimination => "none", OptimizationKind::CountersOnly => "counters", OptimizationKind::BlowUpAnalysis => "blow_up", OptimizationKind::SizeAnalysis => "size", OptimizationKind::PerfOnly => "perf", - OptimizationKind::BottleneckElimination => "ilp", } } } @@ -772,9 +772,7 @@ async fn run_scaling_loop<'a>( ) -> RunMetadata { let output_dir = make_output_dir(&config.name, &config.workload, optimizations.kind.label()); let ir = built.ir(); - if matches!(optimizations.kind, OptimizationKind::PerfOnly) { - save_id(&mut deep_clone(ir), &output_dir.join("id.txt")); - } + save_id(&mut deep_clone(ir), &output_dir.join("id.txt")); let mut final_run_metadata = RunMetadata::default(); let mut best_throughput: usize = 0; let mut no_improvement_count: usize = 0; @@ -848,17 +846,23 @@ async fn run_scaling_loop<'a>( } run_metadata.save_perf(&output_dir, config.num_physical_clients, num_virtual, run); - if run_throughput == 0 { + let unstable_run = run_was_unstable(&run_metadata.throughputs); + if run_throughput == 0 || unstable_run { zero_throughput_count += 1; - println!( - "Zero throughput detected ({}/{})", - zero_throughput_count, NUM_RUNS_NO_THROUGHPUT - ); + if unstable_run { + println!( + "Unstable throughput detected ({}/{})", + zero_throughput_count, NUM_RUNS_NO_THROUGHPUT + ); + } else { + println!( + "Zero throughput detected ({}/{})", + zero_throughput_count, NUM_RUNS_NO_THROUGHPUT + ); + } if zero_throughput_count > NUM_RUNS_NO_THROUGHPUT { return run_metadata; } - } else if run_was_unstable(&run_metadata.throughputs) { - println!("Unstable throughput detected, discarding run"); } else { throughput_sum += run_throughput; successful_runs += 1; @@ -1001,10 +1005,15 @@ async fn run_bottleneck_elimination<'a>( return built; // First ever run — just benchmark the base }; let iteration = latest_iter + 1; + let prev_name = if latest_iter == 0 { + base_name.clone() + } else { + format!("{}_opt{}", base_name, latest_iter) + }; // Early exit if last optimization didn't improve throughput if latest_iter >= 2 { - let thr_prev = max_throughput_for(base, &format!("{}_opt{}", base_name, latest_iter)); + let thr_prev = max_throughput_for(base, &prev_name); let thr_prev_prev = max_throughput_for(base, &format!("{}_opt{}", base_name, latest_iter - 1)); assert!( @@ -1024,11 +1033,6 @@ async fn run_bottleneck_elimination<'a>( } // Find bottleneck from the most recent _none run - let prev_name = if latest_iter == 0 { - base_name.clone() - } else { - format!("{}_opt{}", base_name, latest_iter) - }; let prev_run_dir = find_workload_dirs(base, &prev_name, OptimizationKind::None.label()) .into_iter() .next() @@ -1125,8 +1129,7 @@ async fn run_bottleneck_elimination<'a>( cluster_size, }; - // TODO: Run optimization again once projected CPU is close to 100% - let computed_rewrites = project_total_cpu(&mut ir, &bottleneck_loc, &ilp_inputs, &cycles); + let computed_rewrites = find_optimal_budget(&mut ir, &bottleneck_loc, &ilp_inputs, &cycles); state .cluster_rewrites .insert(bottleneck_cluster.clone(), computed_rewrites); From 45c228fdad6bfdd59366489095e80bfc8b63b327 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Thu, 25 Jun 2026 21:49:35 +0000 Subject: [PATCH 13/29] Partitioning fix (network not counted as relevant operator, filtered out operators that fed into relevant operators but we still needed to know what field to partition on) --- .gitignore | 1 + README.md | 2 +- hydro_optimize/src/decouple_analysis.rs | 159 +++++++++++++++++-- hydro_optimize/src/partition_ilp_analysis.rs | 84 ++-------- setup_gurobi.sh | 8 +- 5 files changed, 160 insertions(+), 94 deletions(-) diff --git a/.gitignore b/.gitignore index 9e4ff31..6b38169 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ scripts/*.png .hydro out*.txt benchmark_results/ +result_backup/ # Neovim stuff tmux* diff --git a/README.md b/README.md index ead191c..7afd702 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ source ~/.zshrc ``` Create a Gurobi license following your organization's instructions. -For free educational licenses, you may request a license from Gurobi, then download it (which should create a gurobi.lic file) and put it on your remote machine under `~/`. +For free educational licenses, you may request a WSL license from Gurobi, then download it (which should create a gurobi.lic file) and put it on your remote machine under `~/`. ## Execution Run all commands below within `tmux` so they continue running even if your SSH connection breaks. diff --git a/hydro_optimize/src/decouple_analysis.rs b/hydro_optimize/src/decouple_analysis.rs index f183f67..ade9c7f 100644 --- a/hydro_optimize/src/decouple_analysis.rs +++ b/hydro_optimize/src/decouple_analysis.rs @@ -221,12 +221,12 @@ fn add_load_to_locations( let res = resource_usages.get_mut(loc).unwrap(); let cpu_temp = std::mem::take(&mut res.cpu); res.cpu = cpu_temp + load.cpu * *var; - let mem_temp = std::mem::take(&mut res.memory); - res.memory = mem_temp + load.memory * *var; - let net_temp = std::mem::take(&mut res.network); - res.network = net_temp + load.network * *var; - let io_temp = std::mem::take(&mut res.io); - res.io = io_temp + load.io * *var; + // let mem_temp = std::mem::take(&mut res.memory); + // res.memory = mem_temp + load.memory * *var; + // let net_temp = std::mem::take(&mut res.network); + // res.network = net_temp + load.network * *var; + // let io_temp = std::mem::take(&mut res.io); + // res.io = io_temp + load.io * *var; } } @@ -401,12 +401,12 @@ fn add_decoupling_overhead( fn add_resource_cost(res: &mut ResourceExpressions, cost: &SarStats, var: Variable) { let cpu_temp = std::mem::take(&mut res.cpu); res.cpu = cpu_temp + (cost.cpu + DECOUPLING_PENALTY) * var; - let mem_temp = std::mem::take(&mut res.memory); - res.memory = mem_temp + cost.memory * var; - let net_temp = std::mem::take(&mut res.network); - res.network = net_temp + cost.network * var; - let io_temp = std::mem::take(&mut res.io); - res.io = io_temp + cost.io * var; + // let mem_temp = std::mem::take(&mut res.memory); + // res.memory = mem_temp + cost.memory * var; + // let net_temp = std::mem::take(&mut res.network); + // res.network = net_temp + cost.network * var; + // let io_temp = std::mem::take(&mut res.io); + // res.io = io_temp + cost.io * var; } /// Only penalize decoupling inner from Tees once per unique location. @@ -809,6 +809,22 @@ pub(crate) fn decouple_analysis( .map(|(expr, cap)| solution.eval(expr.clone()) / cap) .fold(0.0_f64, f64::max); result.location_costs.insert(*loc, max_sat); + + // DEBUG: chosen partition count and per-resource effective saturation for this location. + let chosen_n = is_n_partitions + .get(*loc) + .and_then(|vars| vars.iter().position(|&v| solution.value(v).round() == 1.0)) + .unwrap_or(0); + let per_resource_sat: Vec = res + .iter() + .map(|(expr, cap)| solution.eval(expr.clone()) / cap) + .collect(); + let per_resource_usage: Vec = + res.iter().map(|(expr, _)| solution.eval(expr.clone())).collect(); + println!( + "ILP cost loc {}: chosen_n={}, max_sat={:.4}, effective_usage(cpu,mem,net,io)={:?}, sat={:?}", + loc, chosen_n, max_sat, per_resource_usage, per_resource_sat + ); } for (&op_id, parents) in &op_id_to_parents { @@ -844,11 +860,11 @@ pub(crate) fn decouple_analysis( } // Evaluate per-location partitionability - if let Some(pm) = partition_metadata { + if let Some(partition_metadata) = partition_metadata { for loc in 0..max_num_locations { let num_relevant = solution .eval( - pm.num_relevant_operators + partition_metadata.num_relevant_operators .get(&loc) .cloned() .unwrap_or_default(), @@ -856,7 +872,7 @@ pub(crate) fn decouple_analysis( .round() as i64; let num_partitionable = solution .eval( - pm.partitionable_operators + partition_metadata.partitionable_operators .get(&loc) .cloned() .unwrap_or_default(), @@ -864,7 +880,7 @@ pub(crate) fn decouple_analysis( .round() as i64; let num_persists = solution .eval( - pm.num_persist_operators + partition_metadata.num_persist_operators .get(&loc) .cloned() .unwrap_or_default(), @@ -883,7 +899,7 @@ pub(crate) fn decouple_analysis( // Resolve per-op field choices from the ILP solution. For each op that has field // variables, find which field the solver set to 1 (if any). - for (op_id, fields) in &pm.op_id_to_field_vars { + for (op_id, fields) in &partition_metadata.op_id_to_field_vars { if let Some((name, _)) = fields .iter() .find(|(_, var)| solution.value(**var).round() == 1.0) @@ -909,6 +925,16 @@ pub(crate) fn decouple_analysis( } } + project_location_costs( + ir, + bottleneck, + &op_counts, + &op_sizes, + &network_cost_table, + &per_op_load, + &result, + ); + result } @@ -1036,3 +1062,102 @@ pub(crate) fn project_total_cpu( println!("Projected total CPU usage at bottleneck: {:.2}%", total_cpu); std::process::exit(0); } + +/// Like `project_total_cpu`, but attributes cost to each location AFTER a rewrite. +/// +/// Rules: +/// - An operator executes on the location its parent is assigned to (which equals its own +/// assigned location for non-decoupled ops). Its compute load is charged there. +/// - Networking is special-cased: an existing network charges its send cost (parent's +/// cardinality, larger for broadcasts) to the sender's location and its receive cost +/// (the network's own cardinality) to the receiver's location. A new decoupling edge +/// (`op_to_network`) charges the op's output cost to both the sender and receiver locations. +pub(crate) fn project_location_costs( + ir: &mut [HydroRoot], + bottleneck: &LocationId, + op_counts: &HashMap, + op_sizes: &HashMap, + network_cost_table: &NetworkCostTable, + per_op_load: &HashMap, + rewrite: &Rewrite, +) { + let mut loc_cpu: HashMap = HashMap::new(); + + let net_cpu = |count: usize, size: u64| -> f64 { + if count > 0 && size > 0 { + network_cost_table.network_cost(count, size).cpu + } else { + 0.0 + } + }; + + traverse_dfir( + ir, + |_, _| {}, + |node, _| { + let Some(op_id) = node.op_metadata().id else { + return; + }; + // The op executes where its (first) parent is assigned; fall back to its own assignment. + let parent_id = node.input_metadata().first().and_then(|m| m.op.id); + let exec_loc = parent_id + .and_then(|p| rewrite.op_to_loc.get(&p).copied()) + .or_else(|| rewrite.op_to_loc.get(&op_id).copied()); + + let network_type = get_network_type(node, bottleneck); + if let Some(network_type) = network_type { + // Existing network: send cost on the sender, receive cost on the receiver. + let size = op_sizes.get(&op_id).copied().unwrap_or(0); + if matches!(network_type, NetworkType::Send | NetworkType::SendRecv) { + let send_count = parent_id + .and_then(|p| op_counts.get(&p).copied()) + .unwrap_or(0); + let send_loc = parent_id + .and_then(|p| rewrite.op_to_loc.get(&p).copied()) + .or(exec_loc); + if let Some(l) = send_loc { + *loc_cpu.entry(l).or_default() += net_cpu(send_count, size); + } + } + if matches!(network_type, NetworkType::Recv | NetworkType::SendRecv) { + let recv_count = op_counts.get(&op_id).copied().unwrap_or(0); + let recv_loc = rewrite.op_to_loc.get(&op_id).copied().or(exec_loc); + if let Some(l) = recv_loc { + *loc_cpu.entry(l).or_default() += net_cpu(recv_count, size); + } + } + return; + } + + if node.metadata().location_id.root() != bottleneck { + return; + } + + // New decoupling edge on this op's output: charge serialization to both ends. + if let Some(&(from_loc, to_loc)) = rewrite.op_to_network.get(&op_id) { + let size = op_sizes.get(&op_id).copied().unwrap_or(0); + let count = op_counts.get(&op_id).copied().unwrap_or(0); + let c = net_cpu(count, size); + *loc_cpu.entry(from_loc).or_default() += c; + *loc_cpu.entry(to_loc).or_default() += c; + } + + // Compute load, charged to the op's execution location. + if let Some(load) = per_op_load.get(&op_id) + && let Some(l) = exec_loc + { + *loc_cpu.entry(l).or_default() += load.cpu; + } + }, + ); + + let mut locs: Vec = loc_cpu.keys().copied().collect(); + locs.sort_unstable(); + println!( + "=== Predicted per-location CPU after rewrite ({} locations) ===", + rewrite.num_locations() + ); + for loc in locs { + println!(" Location {}: predicted_cpu={:.2}%", loc, loc_cpu[&loc]); + } +} diff --git a/hydro_optimize/src/partition_ilp_analysis.rs b/hydro_optimize/src/partition_ilp_analysis.rs index 75bfff2..99c0d0d 100644 --- a/hydro_optimize/src/partition_ilp_analysis.rs +++ b/hydro_optimize/src/partition_ilp_analysis.rs @@ -340,72 +340,6 @@ fn create_canonical_fields( op_to_dependencies } -/// Whether a node affects partitionability analysis -pub enum Partitionability { - NoEffect, - Conditional, - Unpartitionable, -} - -/// If this node is an IDB, and it is the only non-network node in its location, can we partition that location? -/// - `NoEffect`: Yes, and we can actually partition randomly for each element. -/// - `Conditional`: Yes, we can partition on some specific keys. -/// - `Unpartitionable`: No. -fn node_partitionability(node: &HydroNode) -> Partitionability { - // Commutative+associative reduces from reduce pushdown are tagged and can be partitioned - if node.op_metadata().cpu_usage == Some(COM_ASSOC_REDUCE_TAG) { - return Partitionability::NoEffect; - } - match node { - HydroNode::Placeholder => { - panic!() - } - HydroNode::Cast { .. } - | HydroNode::ObserveNonDet { .. } - | HydroNode::Source { .. } - | HydroNode::SingletonSource { .. } - | HydroNode::CycleSource { .. } - | HydroNode::Tee { .. } - | HydroNode::Partition { .. } - | HydroNode::YieldConcat { .. } - | HydroNode::BeginAtomic { .. } - | HydroNode::EndAtomic { .. } - | HydroNode::Batch { .. } - | HydroNode::ResolveFutures { .. } - | HydroNode::ResolveFuturesBlocking { .. } - | HydroNode::ResolveFuturesOrdered { .. } - | HydroNode::Filter { .. } - | HydroNode::DeferTick { .. } - | HydroNode::Inspect { .. } - | HydroNode::ExternalInput { .. } - | HydroNode::Network { .. } - | HydroNode::Counter { .. } - | HydroNode::Map { .. } - | HydroNode::FilterMap { .. } - | HydroNode::FlatMap { .. } - | HydroNode::FlatMapStreamBlocking { .. } - | HydroNode::Chain { .. } - | HydroNode::ChainFirst { .. } - | HydroNode::MergeOrdered { .. } => Partitionability::NoEffect, - HydroNode::Join { .. } - | HydroNode::JoinHalf { .. } - | HydroNode::Difference { .. } - | HydroNode::AntiJoin { .. } - | HydroNode::Unique { .. } - | HydroNode::FoldKeyed { .. } - | HydroNode::ReduceKeyed { .. } - | HydroNode::ReduceKeyedWatermark { .. } => Partitionability::Conditional, - HydroNode::CrossProduct { .. } - | HydroNode::CrossSingleton { .. } - | HydroNode::Enumerate { .. } - | HydroNode::Sort { .. } - | HydroNode::Scan { .. } - | HydroNode::ScanAsyncBlocking { .. } - | HydroNode::Fold { .. } - | HydroNode::Reduce { .. } => Partitionability::Unpartitionable, - } -} - /// Whether this node introduces persistence. /// Note: Must keep in sync with `emit_core` in hydro_lang. fn node_persists(node: &HydroNode) -> bool { @@ -672,6 +606,16 @@ fn constrain_field_vars_to_parents( if parents.is_empty() { // Network nodes with no parents: field vars are unconstrained (free to be 0 or 1). // The solver will set them to 1 if doing so helps enable partitioning downstream. + // Add to partitionable operators as always partitionable + let DecoupleILPMetadata { + op_id_to_var, + .. + } = &mut *decoupling_metadata.borrow_mut(); + for (loc, partitionable_expr) in partitionable_operators { + let is_at_loc = op_id_to_var.get(&op_id).unwrap().get(loc).unwrap(); + let temp_expr = std::mem::take(partitionable_expr); + *partitionable_expr = temp_expr + is_at_loc; + } return; } @@ -794,14 +738,10 @@ fn partition_ilp_node_analysis( return; } - let affect_partitionability = match node_partitionability(node) { - Partitionability::NoEffect => false, - Partitionability::Conditional | Partitionability::Unpartitionable => true, - }; // TODO: Known inaccuracy: if a node is not an IDB but flows into the positive edge of a // negation (e.g. an anti-join), then it should also be relevant (and partitioned) to avoid // producing too many outputs - let is_relevant = idbs.contains(&op_id) && affect_partitionability; + let is_relevant = idbs.contains(&op_id); let PartitionILPMetadata { num_relevant_operators, @@ -809,7 +749,7 @@ fn partition_ilp_node_analysis( .. } = metadata; - // A node is relevant if it is an IDB and affects partitionability + // A node is relevant if it is an IDB if is_relevant { add_op_to_location_sum(op_id, decoupling_metadata, num_relevant_operators); } diff --git a/setup_gurobi.sh b/setup_gurobi.sh index fe7d0b0..075f6c3 100755 --- a/setup_gurobi.sh +++ b/setup_gurobi.sh @@ -4,7 +4,7 @@ set -euo pipefail cd /opt sudo wget https://packages.gurobi.com/13.0/gurobi13.0.2_linux64.tar.gz sudo tar xvfz gurobi13.0.2_linux64.tar.gz -echo 'export GUROBI_HOME="/opt/gurobi1302/linux64"' >> ~/.zshrc -echo 'export PATH="${PATH}:${GUROBI_HOME}/bin"' >> ~/.zshrc -echo 'export LD_LIBRARY_PATH="${GUROBI_HOME}/lib"' >> ~/.zshrc -source ~/.zshrc +echo 'export GUROBI_HOME="/opt/gurobi1302/linux64"' >> ~/.bashrc +echo 'export PATH="${PATH}:${GUROBI_HOME}/bin"' >> ~/.bashrc +echo 'export LD_LIBRARY_PATH="${GUROBI_HOME}/lib"' >> ~/.bashrc +source ~/.bashrc From 144800ec6e7fedb8fcd8ccdd3ad1e95a21d13290 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Thu, 25 Jun 2026 21:50:15 +0000 Subject: [PATCH 14/29] Fix CLUSTER_SELF_IDs inserted for decoupling + partitioning --- hydro_optimize/src/rewriter.rs | 13 +++++++++---- hydro_optimize/src/rewrites.rs | 8 +++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/hydro_optimize/src/rewriter.rs b/hydro_optimize/src/rewriter.rs index 979fa53..cb6ab98 100644 --- a/hydro_optimize/src/rewriter.rs +++ b/hydro_optimize/src/rewriter.rs @@ -608,11 +608,16 @@ fn replace_cluster_self_id( ); }, |node, _| { - let Some(op_id) = node.op_metadata().id else { - return; + let target_loc_idx = if let Some(op_id) = node.op_metadata().id { + execution_loc_of_op(op_id, node, rewrite).unwrap_or(0) + } else { + // New op but may still need rewriting if inserted for decoupling + partitioning + locations_map + .iter() + .find(|(_, loc)| **loc == node.metadata().location_id) + .map(|(idx, _)| *idx) + .unwrap_or(0) }; - - let target_loc_idx = execution_loc_of_op(op_id, node, rewrite).unwrap_or(0); replace_cluster_self_id_node( &mut |f| node.visit_debug_expr(f), rewrite, diff --git a/hydro_optimize/src/rewrites.rs b/hydro_optimize/src/rewrites.rs index 57d6210..ba1be46 100644 --- a/hydro_optimize/src/rewrites.rs +++ b/hydro_optimize/src/rewrites.rs @@ -30,15 +30,17 @@ impl VisitMut for ClusterSelfIdReplace { if let syn::Expr::Path(path_expr) = expr { for segment in path_expr.path.segments.iter_mut() { let ident = segment.ident.to_string(); - let prefix = format!("__hydro_lang_cluster_self_id_{}", self.orig_cluster_id); - if ident.starts_with(&prefix) { - if self.orig_cluster_id != self.new_cluster_id { + let orig_prefix = format!("__hydro_lang_cluster_self_id_{}", self.orig_cluster_id); + let new_prefix = format!("__hydro_lang_cluster_self_id_{}", self.new_cluster_id); + if ident.starts_with(&orig_prefix) || ident.starts_with(&new_prefix) { + if ident.starts_with(&orig_prefix) && self.orig_cluster_id != self.new_cluster_id { segment.ident = syn::Ident::new( &format!("__hydro_lang_cluster_self_id_{}", self.new_cluster_id), segment.ident.span(), ); println!("Decoupling: Replaced CLUSTER_SELF_ID"); } + // Also consider new_prefix, for CLUSTER_SELF_IDs inserted after decoupling that still need to be partitioned if self.num_partitions > 0 { let num_partitions = self.num_partitions; let expr_content = std::mem::replace(expr, syn::Expr::PLACEHOLDER); From 9152f3902397ca3b5116ae05f515e7a48128d029 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Sat, 27 Jun 2026 00:27:05 +0000 Subject: [PATCH 15/29] Fix clients considered bottleneck, calibration after optimization --- hydro_optimize/src/deploy_and_analyze.rs | 37 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index fb1ee9e..4297fe9 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -40,8 +40,7 @@ const SAR_USAGE_PREFIX: &str = "HYDRO_OPTIMIZE_SAR:"; const LATENCY_PREFIX: &str = "HYDRO_OPTIMIZE_LAT:"; const THROUGHPUT_PREFIX: &str = "HYDRO_OPTIMIZE_THR:"; -pub const MAX_BUDGET_PER_CLUSTER: usize = 7; -pub const MAX_OPTIMIZATION_ITERATIONS_PAST_NO_IMPROVEMENT: usize = 2; +pub const MAX_BUDGET_PER_CLUSTER: usize = 10; const BIMODAL_CV_THRESHOLD: f64 = 0.3; /// Returns true if min throughput in the measurement window is significantly lower than the max overall @@ -1032,13 +1031,22 @@ async fn run_bottleneck_elimination<'a>( } } - // Find bottleneck from the most recent _none run + let excluded_names: HashSet = config + .optimizations + .exclude + .iter() + .filter_map(|loc| config.location_id_to_cluster.get(loc).cloned()) + .collect(); let prev_run_dir = find_workload_dirs(base, &prev_name, OptimizationKind::None.label()) .into_iter() .next() .unwrap_or_else(|| panic!("No _none run found for '{}'", prev_name)); - let bottleneck_name = - find_bottleneck_from_run(&prev_run_dir, AWS_NETWORK_BYTES_PER_SEC, AWS_IO_TPS); + let bottleneck_name = find_bottleneck_from_run( + &prev_run_dir, + AWS_NETWORK_BYTES_PER_SEC, + AWS_IO_TPS, + &excluded_names, + ); let bottleneck_cluster = original_cluster_name(&bottleneck_name).to_string(); if state.is_exhausted(&bottleneck_cluster) { @@ -1053,6 +1061,13 @@ async fn run_bottleneck_elimination<'a>( if !state.cluster_rewrites.contains_key(&bottleneck_cluster) { let analysis_name; let analysis_built; + // Snapshot the base clusters/loc-map. When prior rewrites are replayed below, + // they create new (decoupled/partitioned) clusters that the analysis passes must + // deploy. Those passes deploy with `config.clusters`/`config.location_id_to_cluster`, + // so we apply the rewrites directly into `config` here, then restore the base + // snapshots before the final apply recreates them from scratch. + let base_clusters = config.clusters.clone(); + let base_loc_map = config.location_id_to_cluster.clone(); if state.applied.is_empty() { analysis_name = base_name.clone(); analysis_built = None; @@ -1062,15 +1077,13 @@ async fn run_bottleneck_elimination<'a>( analysis_name = name; let rewrites = state.rewrites_to_apply(); - let mut analysis_clusters = config.clusters.clone(); - let mut analysis_loc_map = config.location_id_to_cluster.clone(); let mut builder = FlowBuilder::from_built(&built); builder.replace_ir(deep_clone(built.ir())); analysis_built = Some(apply_loaded_rewrites( builder.finalize(), - &mut analysis_clusters, + &mut config.clusters, &rewrites, - &mut analysis_loc_map, + &mut config.location_id_to_cluster, )); } @@ -1133,6 +1146,12 @@ async fn run_bottleneck_elimination<'a>( state .cluster_rewrites .insert(bottleneck_cluster.clone(), computed_rewrites); + + // Restore the base clusters/loc-map: the final apply below recreates the + // decoupled clusters from the base program, so `config` must not already + // contain the analysis-pass copies (which would duplicate them). + config.clusters = base_clusters; + config.location_id_to_cluster = base_loc_map; } // Apply next budget for the bottleneck cluster From a55c36f0c8d5fe0091efca3d46f85c3ce6d64e40 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Sat, 27 Jun 2026 00:27:24 +0000 Subject: [PATCH 16/29] Fix clients detected as bottleneck --- hydro_optimize/src/parse_results.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index d3d2310..bd4918d 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -23,7 +23,12 @@ use crate::deploy_and_analyze::{ /// 1. Finds the run suffix (e.g. "10c_500vc_r0") with highest avg throughput. /// 2. For that run, reads all cluster CSVs and returns the cluster name whose SAR stats /// are closest to saturation (100% CPU, 100% memory, or at/above max network/IO). -pub fn find_bottleneck_from_run(run_dir: &Path, network_bytes_per_sec: f64, io_tps: f64) -> String { +pub fn find_bottleneck_from_run( + run_dir: &Path, + network_bytes_per_sec: f64, + io_tps: f64, + excluded_names: &HashSet, +) -> String { let csv_re = Regex::new(r"^(.+)_(\d+c_\d+vc_r\d+)\.csv$").unwrap(); // Group CSVs by suffix @@ -72,14 +77,18 @@ pub fn find_bottleneck_from_run(run_dir: &Path, network_bytes_per_sec: f64, io_t .fold(0.0_f64, f64::max) }; + // Excluded clusters/processes (e.g. clients) are never optimization targets, so they + // are not eligible to be the bottleneck. Compare against the base cluster name so that + // decoupled/partitioned variants (e.g. "client_loc1") are excluded too. let (bottleneck_name, _) = suffix_to_files[&best_suffix] .iter() + .filter(|(cluster_name, _)| !excluded_names.contains(original_cluster_name(cluster_name))) .map(|(cluster_name, path)| { let avg_sar = csv_avg_sar(path); (cluster_name.clone(), score(&avg_sar)) }) .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap()) - .expect("No cluster CSVs for best suffix"); + .expect("No non-excluded cluster CSVs for best suffix"); println!( "Bottleneck from run: {} (suffix: {})", From 3d5c0a306485f2c5edd49af8af0afff4e002251e Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Sat, 27 Jun 2026 00:27:47 +0000 Subject: [PATCH 17/29] Fix partitioning for existing networks where output type and input type differ --- hydro_optimize/src/rewriter.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hydro_optimize/src/rewriter.rs b/hydro_optimize/src/rewriter.rs index cb6ab98..cf30fda 100644 --- a/hydro_optimize/src/rewriter.rs +++ b/hydro_optimize/src/rewriter.rs @@ -83,6 +83,14 @@ fn map_before_network(node: &mut HydroNode, network_metadata: &NetworkMetadata) if let Some(field) = &network_metadata.partition_field { // If partitioning and there is a field to hash on, use it let struct_or_tuple: syn::Expr = syn::parse_quote! { struct_or_tuple }; + // If this is an existing network, then the index is over fields of the network after the receipt(sender, T), but before the send, the type is just T, so lop off the 1st layer of the field. + let field: &[String] = if !network_metadata.new { + // Partitioning on sender location CAN be supported, but it will involve introducing a Hash over CLUSTER_SELF_ID + assert_eq!(field[0], "1", "Unsupported: Partitioning on sender location."); + &field[1..] + } else { + field + }; let struct_or_tuple_with_fields = StructOrTuple::to_syn_expr(struct_or_tuple, field); syn::parse_quote!({ let mut s = ::std::hash::DefaultHasher::new(); From 23d9e642ced6d8eef655b962442b9926e9818860 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Sat, 27 Jun 2026 00:28:02 +0000 Subject: [PATCH 18/29] Run rewrites up to budget 10 --- hydro_optimize/src/decouple_analysis.rs | 36 +++++++++---------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/hydro_optimize/src/decouple_analysis.rs b/hydro_optimize/src/decouple_analysis.rs index ade9c7f..6dc5a61 100644 --- a/hydro_optimize/src/decouple_analysis.rs +++ b/hydro_optimize/src/decouple_analysis.rs @@ -5,9 +5,7 @@ use std::time::Instant; use serde::{Deserialize, Serialize}; use crate::deploy::{AWS_IO_TPS, AWS_NETWORK_BYTES_PER_SEC}; -use crate::deploy_and_analyze::{ - MAX_BUDGET_PER_CLUSTER, MAX_OPTIMIZATION_ITERATIONS_PAST_NO_IMPROVEMENT, -}; +use crate::deploy_and_analyze::MAX_BUDGET_PER_CLUSTER; use crate::parse_results::{NetworkCostTable, SarStats}; use crate::partition_ilp_analysis::{apply_budget_constraints, partition_ilp_analysis}; use crate::partition_syn_analysis::StructOrTupleIndex; @@ -37,7 +35,6 @@ pub fn num_to_alpha(n: usize) -> String { // Penalty for decoupling regardless of cardinality (to prevent decoupling low cardinality operators) const DECOUPLING_PENALTY: f64 = 0.0001; -const IMPROVEMENT_THRESHOLD: f64 = 0.01; // Minimum improvement to keep increasing budget const LEXICOGRAPHIC_EPSILON: f64 = 0.0001; // Tiebreaker weight to minimize non-bottleneck locations /// Each operator is assigned either 0 or 1 @@ -819,8 +816,10 @@ pub(crate) fn decouple_analysis( .iter() .map(|(expr, cap)| solution.eval(expr.clone()) / cap) .collect(); - let per_resource_usage: Vec = - res.iter().map(|(expr, _)| solution.eval(expr.clone())).collect(); + let per_resource_usage: Vec = res + .iter() + .map(|(expr, _)| solution.eval(expr.clone())) + .collect(); println!( "ILP cost loc {}: chosen_n={}, max_sat={:.4}, effective_usage(cpu,mem,net,io)={:?}, sat={:?}", loc, chosen_n, max_sat, per_resource_usage, per_resource_sat @@ -864,7 +863,8 @@ pub(crate) fn decouple_analysis( for loc in 0..max_num_locations { let num_relevant = solution .eval( - partition_metadata.num_relevant_operators + partition_metadata + .num_relevant_operators .get(&loc) .cloned() .unwrap_or_default(), @@ -872,7 +872,8 @@ pub(crate) fn decouple_analysis( .round() as i64; let num_partitionable = solution .eval( - partition_metadata.partitionable_operators + partition_metadata + .partitionable_operators .get(&loc) .cloned() .unwrap_or_default(), @@ -880,7 +881,8 @@ pub(crate) fn decouple_analysis( .round() as i64; let num_persists = solution .eval( - partition_metadata.num_persist_operators + partition_metadata + .num_persist_operators .get(&loc) .cloned() .unwrap_or_default(), @@ -949,7 +951,6 @@ pub fn find_optimal_budget( cycle_source_to_sink_parent: &HashMap, ) -> Vec { let mut results = Vec::new(); - let mut no_improvement = 0; for budget in 2..=MAX_BUDGET_PER_CLUSTER { let start = Instant::now(); @@ -969,20 +970,7 @@ pub fn find_optimal_budget( rewrite.num_partitions, start.elapsed() ); - - let improved = results - .last() - .is_none_or(|prev: &Rewrite| prev.max_cost() - cost > IMPROVEMENT_THRESHOLD); - - if improved { - no_improvement = 0; - results.push(rewrite); - } else { - no_improvement += 1; - if no_improvement >= MAX_OPTIMIZATION_ITERATIONS_PAST_NO_IMPROVEMENT { - break; - } - } + results.push(rewrite); } results From 4bd4e1faa6427d83d0d1765a643911ec3a014b98 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Sat, 27 Jun 2026 00:28:21 +0000 Subject: [PATCH 19/29] Fix partitioning existing network --- hydro_optimize/src/partition_syn_analysis.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydro_optimize/src/partition_syn_analysis.rs b/hydro_optimize/src/partition_syn_analysis.rs index 9b4b604..48ab267 100644 --- a/hydro_optimize/src/partition_syn_analysis.rs +++ b/hydro_optimize/src/partition_syn_analysis.rs @@ -422,7 +422,7 @@ impl StructOrTuple { } } - pub fn to_syn_expr(mut tuple: syn::Expr, indices: &StructOrTupleIndex) -> syn::Expr { + pub fn to_syn_expr(mut tuple: syn::Expr, indices: &[String]) -> syn::Expr { for index in indices { let member = if let Ok(num_index) = index.parse::() { syn::Member::Unnamed(syn::Index::from(num_index)) From 3b604dfd70dd07be6e7a77a9bc08c963d5154e33 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Sat, 27 Jun 2026 01:22:50 +0000 Subject: [PATCH 20/29] CAS fix write throughput 0 for write-only workloads with more clients (writes overridden by None values from election) --- .../src/compare_and_swap/cas_distributed.rs | 206 ++++++++---------- 1 file changed, 90 insertions(+), 116 deletions(-) diff --git a/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs b/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs index 4fff00c..43a343e 100644 --- a/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs +++ b/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs @@ -9,7 +9,6 @@ use hydro_lang::{ nondet::nondet, prelude::{Bounded, Cluster, KeyedStream, Optional, Singleton, Stream, TCP, Tick, Unbounded}, }; -use hydro_std::membership::track_membership; use serde::{Deserialize, Serialize}; use stageleft::q; use std::{fmt::Debug, hash::Hash, time::Duration}; @@ -80,7 +79,7 @@ impl Ord for Request { pub struct Response { pub request: Request, pub max_ballot: Ballot, - pub state: Option>, + pub state_and_ballot: Option<(CASState, Ballot)>, } impl<'a, 'b> DistributedCAS<'a, 'b> { @@ -108,69 +107,6 @@ impl<'a, 'b> DistributedCAS<'a, 'b> { } } - /// Send the message over the network to all other replicas, but just pass the message onwards for ourselves. - fn smart_broadcast( - &self, - stream: Stream, Unbounded, NoOrder>, - ) -> KeyedStream, Payload, Cluster<'a, Replica>, Unbounded, NoOrder> - where - Payload: Clone + Serialize + for<'de> Deserialize<'de> + 'a, - { - let nondet_membership = nondet!(/** Membership is static */); - let members = self.cluster.source_cluster_members(self.cluster); - let curr_replicas = track_membership(members); - - let stream_with_dest = sliced! { - let stream = use(stream.clone(), nondet_membership); - let curr_replicas = use(curr_replicas, nondet_membership); - - curr_replicas - .into_keyed_stream() - .entries() - .filter_map(q!(move |(member_id, present)| (present && member_id != CLUSTER_SELF_ID).then_some(member_id))) - .cross_product(stream) - }; - let sent_stream = stream_with_dest.demux(self.cluster, TCP.fail_stop().bincode()); - - // Add a stream to ourselves but not over the network - let tick = self.cluster.tick(); - let stream_to_self = stream - .map(q!(move |payload| (CLUSTER_SELF_ID.clone(), payload))) - .into_keyed() - .batch( - &tick, - nondet!(/** Only exists to defer_tick and avoid cycle with negation */), - ) - .defer_tick() - .all_ticks(); - sent_stream.merge_unordered(stream_to_self) - } - - /// Similar to `smart_broadcast`, but for sending to a single destination. - fn smart_demux( - &self, - stream: Stream<(MemberId, Payload), Cluster<'a, Replica>, Unbounded, NoOrder>, - ) -> KeyedStream, Payload, Cluster<'a, Replica>, Unbounded, NoOrder> - where - Payload: Clone + Serialize + for<'de> Deserialize<'de> + 'a, - { - let (should_pass, should_demux) = - stream.partition(q!( - move |(member_id, _payload)| *member_id == CLUSTER_SELF_ID - )); - let demuxed = should_demux.demux(self.cluster, TCP.fail_stop().bincode()); - let tick = self.cluster.tick(); - should_pass - .into_keyed() - .batch( - &tick, - nondet!(/** Only exists to defer_tick and avoid cycle with negation */), - ) - .defer_tick() - .all_ticks() - .merge_unordered(demuxed) - } - /// Returns the state associated with the respondent with the highest ballot, and whether all agree /// Returns only 1 response per request ID, on the tick the quorum is first reached. #[expect(clippy::type_complexity, reason = "Stream type")] @@ -191,7 +127,6 @@ impl<'a, 'b> DistributedCAS<'a, 'b> { let f = self.f; sliced! { let responses = use(responses.entries(), nondet!(/** The order in which inputs are collected at the quorum affects what is outputted */)); - let mut reached_quorum = use::state_null::, Bounded, NoOrder>>(); let mut prev_responses = use::state_null::), Tick<_>, Bounded, NoOrder>>(); let current_responses = prev_responses.clone().chain(responses).into_keyed(); @@ -199,27 +134,34 @@ impl<'a, 'b> DistributedCAS<'a, 'b> { let (no_quorum, quorum) = count_per_key .entries() - .partition(q!(move |(_request_id, count)| *count < f + 1)); - let (just_quorum, all_quorum) = quorum .partition(q!(move |(_request_id, count)| *count < 2 * f + 1)); let just_reached_quorum = current_responses .clone() - .filter_key_not_in(no_quorum.into_keyed().keys()) - .filter_key_not_in(reached_quorum); + .filter_key_not_in(no_quorum.into_keyed().keys()); let result = just_reached_quorum .map(q!(|response| (response, true))) .reduce(q!(|(prev_response, all_agree), (response, _)| { - if prev_response.state != response.state { + if prev_response.state_and_ballot.as_ref().map(|(state, _)| state) != response.state_and_ballot.as_ref().map(|(state, _)| state) { *all_agree = false; } + // Return the state with the largest accepted ballot + if let Some((_, prev_ballot)) = &prev_response.state_and_ballot { + if let Some((_, ballot)) = &response.state_and_ballot + && *prev_ballot < *ballot { + prev_response.state_and_ballot = response.state_and_ballot; + } + } + else { + prev_response.state_and_ballot = response.state_and_ballot; + } + // Return the largest promised ballot. Note: This is just the ballot the leader needs to exceed to win writes; it does not indicate which write is freshest if prev_response.max_ballot < response.max_ballot { - *prev_response = response; + prev_response.max_ballot = response.max_ballot; } }, commutative = manual_proof!(/** Max is order independent. No two writes will use the same ballot. */))); - reached_quorum = just_quorum.into_keyed().keys(); - prev_responses = current_responses.filter_key_not_in(all_quorum.into_keyed().keys()).entries(); + prev_responses = current_responses.filter_key_not_in(quorum.into_keyed().keys()).entries(); result.values() } @@ -323,8 +265,10 @@ where num: 0, node: CLUSTER_SELF_ID.clone(), }))); - let mut state = use::state_null::, _, Bounded>>(); + let mut state = use::state_null::, Ballot), _, Bounded>>(); // Last state where we know a majority of replicas agreed. + // NOTE: The ballot here is the request ballot sent to check the state, not the one associated with the accepted state. + // It should only be used to check if the committed state was sent in response to some blocked request. let mut committed_state = use::state_null::>), _, Bounded>>(); // Writes blocking in reconcilation or election let mut write_queue = use::state_null::, CASState), _, Bounded, NoOrder>>(); @@ -375,10 +319,10 @@ where .partition(q!(|(_response, all_agree)| *all_agree)); // Propose reconciling write if some replicas disagree let proposed_reconciling_write = won_election_write_uncommitted - .map(q!(|(response, _all_agree)| (response.request.id, (None, response.state.expect("uncommitted election response missing state"))))); + .map(q!(|(response, _all_agree)| (response.request.id, (None, response.state_and_ballot.map(|(state, _ballot)| state).expect("uncommitted election response missing state"))))); let new_committed_state = won_election_write_complete .clone() - .map(q!(|(response, _all_agree)| (response.max_ballot, response.state))) + .map(q!(|(response, _all_agree)| (response.max_ballot, response.state_and_ballot.map(|(state, _ballot)| state)))) .chain(committed_state.into_stream()) .max(); // Propose a valid client write if the election is succesful and all agree @@ -388,7 +332,7 @@ where .cross_product(won_election_write_complete) .filter_map(q!(|((request_id, (client_id, state)), (response, _all_agree))| // Allow write if version is prev+1 or the writer is the same and the version hasn't changed - if response.state.is_none_or(|curr_state| curr_state.version + 1 == state.version || + if response.state_and_ballot.is_none_or(|(curr_state, _curr_ballot)| curr_state.version + 1 == state.version || (curr_state.writer == state.writer && curr_state.version == state.version)) { Some((request_id, (Some(client_id), state))) } @@ -462,12 +406,22 @@ where let winning_write = incoming_requests .clone() .values() - .sort() - .last() - .filter_map(q!(|request| request.ballot.zip(request.state))); + .reduce(q!(|max_req, request| { + // Find the max ballot request with state + if let Some(old_ballot) = &max_req.ballot { + if let Some(ref new_ballot) = request.ballot && request.state.is_some() && *old_ballot <= *new_ballot { + *max_req = request; + } + } + else { + *max_req = request; + } + }, commutative = manual_proof!(/** Max is commutative */))) + .filter(q!(|request| request.ballot.is_some() && request.state.is_some())); let next_state = winning_write .zip(max_ballot.clone()) - .filter_map(q!(|((write_ballot, state), ballot)| (write_ballot >= ballot).then_some(state))) + .filter_map(q!(|(winning_request, ballot)| + (*winning_request.ballot.as_ref().unwrap() >= ballot).then_some((winning_request.state.unwrap(), winning_request.ballot.unwrap())))) .or(state); // Attach ballots to requests and responses @@ -494,50 +448,64 @@ where // ------------------------------------------------------------- // Broadcast requests. // ------------------------------------------------------------- - let sent_requests = self.smart_broadcast( - client_reads - .entries() - .map(q!(|(client_id, request_id)| Request { - id: request_id, - client_id: Some(client_id), - ballot: None, - state: None, - })) - .merge_unordered(election_requests.map(q!(|ballot| Request { - id: UniqueRequestId::generate(), - client_id: None, - ballot: Some(ballot), - state: None, - }))) - .merge_unordered(write_requests.map(q!(|( - (request_id, (client_id, write)), - ballot, - )| Request { - id: request_id, - client_id, - ballot: Some(ballot), - state: Some(write), - }))), - ); + let sent_reads = client_reads + .entries() + .map(q!(|(client_id, request_id)| Request { + id: request_id, + client_id: Some(client_id), + ballot: None, + state: None, + })) + .broadcast( + self.cluster, + TCP.fail_stop().bincode(), + nondet!(/** static membership */), + ); + let sent_elections = election_requests + .map(q!(|ballot| Request { + id: UniqueRequestId::generate(), + client_id: None, + ballot: Some(ballot), + state: None, + })) + .broadcast( + self.cluster, + TCP.fail_stop().bincode(), + nondet!(/** static membership */), + ); + let sent_writes = write_requests + .map(q!(|((request_id, (client_id, write)), ballot)| Request { + id: request_id, + client_id, + ballot: Some(ballot), + state: Some(write), + })) + .broadcast( + self.cluster, + TCP.fail_stop().bincode(), + nondet!(/** static membership */), + ); + let sent_requests = sent_reads + .merge_unordered(sent_elections) + .merge_unordered(sent_writes); incoming_requests_complete.complete(sent_requests); // ------------------------------------------------------------- // Reply to requests with responses. // ------------------------------------------------------------- - let sent_responses = - self.smart_demux(responses.entries().map(q!(|( - sender, - ((request, max_ballot), state), - )| { + let sent_responses = responses + .entries() + .map(q!(|(sender, ((request, max_ballot), state_and_ballot))| { ( sender, Response { request, max_ballot, - state, + state_and_ballot, }, ) - }))) + })) + .demux(self.cluster, TCP.fail_stop().bincode()) .values() .map(q!(|response| (response.request.id, response))) .into_keyed(); @@ -560,7 +528,10 @@ where .request .client_id .expect("read result missing client_id"), - (response.request.id, response.state.clone()) + ( + response.request.id, + response.state_and_ballot.map(|(state, _ballot)| state) + ) )))) .merge_unordered(unblocked_reads) .demux(sender, TCP.fail_stop().bincode()) @@ -580,7 +551,10 @@ where .client_id .expect("write success missing client_id"), response.request.id, - response.state.clone().expect("write success missing state") + response + .state_and_ballot + .map(|(state, _ballot)| state) + .expect("write success missing state") )))); let write_processed = write_success .clone() From 9db318e6c1cf2f8831881cace4df74a9cd5466bc Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 30 Jun 2026 06:18:08 +0000 Subject: [PATCH 21/29] No EDB decoupling --- hydro_optimize/src/decouple_analysis.rs | 21 ++++- hydro_optimize/src/partition_ilp_analysis.rs | 73 +----------------- hydro_optimize/src/partition_node_analysis.rs | 21 +---- hydro_optimize/src/rewrites.rs | 76 +++++++++++++++++++ 4 files changed, 97 insertions(+), 94 deletions(-) diff --git a/hydro_optimize/src/decouple_analysis.rs b/hydro_optimize/src/decouple_analysis.rs index 6dc5a61..543faa0 100644 --- a/hydro_optimize/src/decouple_analysis.rs +++ b/hydro_optimize/src/decouple_analysis.rs @@ -9,7 +9,7 @@ use crate::deploy_and_analyze::MAX_BUDGET_PER_CLUSTER; use crate::parse_results::{NetworkCostTable, SarStats}; use crate::partition_ilp_analysis::{apply_budget_constraints, partition_ilp_analysis}; use crate::partition_syn_analysis::StructOrTupleIndex; -use crate::rewrites::{get_tick_id, is_serializable, is_syntactic_sugar, op_id_to_parents}; +use crate::rewrites::{all_inputs, get_tick_id, is_serializable, is_syntactic_sugar, nodes_dependent_on_inputs, op_id_to_parents}; use good_lp::solvers::lp_solvers::{GurobiSolver, LpSolution, LpSolver}; use good_lp::{ Constraint, Expression, ProblemVariables, Solution, SolverModel, Variable, constraint, @@ -736,6 +736,13 @@ pub(crate) fn decouple_analysis( network_ids: HashMap::new(), }); let op_id_to_parents = op_id_to_parents(ir, Some(bottleneck), cycle_source_to_sink_parent); + let inputs = all_inputs(ir, &decoupling_metadata.borrow().bottleneck); + let idbs = nodes_dependent_on_inputs( + ir, + &decoupling_metadata.borrow().bottleneck, + &inputs, + &op_id_to_parents, + ); let bottleneck_ops: RefCell> = RefCell::new(HashSet::new()); traverse_dfir( @@ -752,16 +759,23 @@ pub(crate) fn decouple_analysis( decouple_analysis_node(node, next_op_id, &op_id_to_parents, &decoupling_metadata); }, ); - for parents in op_id_to_parents.values() { + for (op_id, parents) in &op_id_to_parents { let DecoupleILPMetadata { op_id_to_var, variables, constraints, .. } = &mut *decoupling_metadata.borrow_mut(); + + let mut same_output_loc = parents.clone(); + if !idbs.contains(op_id) { + // No decoupling EDBs, otherwise broadcast cluster members might be decoupled from the broadcast itself, and the membership might not arrive in time for the broadcast + same_output_loc.push(*op_id); + } + // Add parent constraints. All parents of an op must output to the same machine (be assigned the same var) add_equality_constr( - parents, + &same_output_loc, max_num_locations, op_id_to_var, variables, @@ -774,6 +788,7 @@ pub(crate) fn decouple_analysis( Some(partition_ilp_analysis( ir, &op_id_to_parents, + idbs, &decoupling_metadata, )) } else { diff --git a/hydro_optimize/src/partition_ilp_analysis.rs b/hydro_optimize/src/partition_ilp_analysis.rs index 99c0d0d..8cfd83a 100644 --- a/hydro_optimize/src/partition_ilp_analysis.rs +++ b/hydro_optimize/src/partition_ilp_analysis.rs @@ -10,70 +10,11 @@ use hydro_lang::{ }; use syn::visit::Visit; -use crate::reduce_pushdown::COM_ASSOC_REDUCE_TAG; use crate::{ decouple_analysis::{DecoupleILPMetadata, ResourceExpressions, num_to_alpha}, - partition_node_analysis::all_inputs, partition_syn_analysis::{AnalyzeClosure, StructOrTuple, StructOrTupleIndex}, }; -/// Add id to dependent_nodes if its parent_id is already in dependent_nodes -fn insert_dependent_node( - dependent_nodes: &RefCell>, - parent_id: Option, - id: usize, -) { - let mut dependent_nodes_borrow = dependent_nodes.borrow_mut(); - if let Some(parent_id) = parent_id - && dependent_nodes_borrow.contains(&parent_id) - { - dependent_nodes_borrow.insert(id); - } -} - -/// Returns IDs of nodes that are dependent on at least 1 input -fn nodes_dependent_on_inputs( - ir: &mut [HydroRoot], - location: &LocationId, - inputs: &[usize], - op_to_parents: &HashMap>, -) -> HashSet { - let dependent_nodes = RefCell::new(HashSet::from_iter(inputs.iter().cloned())); - let mut num_dependent_nodes = inputs.len(); - - loop { - traverse_dfir( - ir, - |root, next_stmt_id| { - if root.input_metadata().location_id.root() == location { - insert_dependent_node( - &dependent_nodes, - root.input_metadata().op.id, - *next_stmt_id, - ); - } - }, - |node, next_stmt_id| { - if node.metadata().location_id.root() == location.root() - && let Some(parents) = op_to_parents.get(next_stmt_id) - { - for parent_id in parents { - insert_dependent_node(&dependent_nodes, Some(*parent_id), *next_stmt_id); - } - } - }, - ); - - if dependent_nodes.borrow().len() == num_dependent_nodes { - // No new dependent nodes found, reached fixpoint - break; - } - num_dependent_nodes = dependent_nodes.borrow().len(); - } - - dependent_nodes.take() -} - /// Given a node type, return how its output fields is dependent on its parents. /// Must contain an entry for each parent, even if there is no dependency. /// Contains 1 entry if there are no parents. @@ -607,10 +548,7 @@ fn constrain_field_vars_to_parents( // Network nodes with no parents: field vars are unconstrained (free to be 0 or 1). // The solver will set them to 1 if doing so helps enable partitioning downstream. // Add to partitionable operators as always partitionable - let DecoupleILPMetadata { - op_id_to_var, - .. - } = &mut *decoupling_metadata.borrow_mut(); + let DecoupleILPMetadata { op_id_to_var, .. } = &mut *decoupling_metadata.borrow_mut(); for (loc, partitionable_expr) in partitionable_operators { let is_at_loc = op_id_to_var.get(&op_id).unwrap().get(loc).unwrap(); let temp_expr = std::mem::take(partitionable_expr); @@ -891,6 +829,7 @@ fn calculate_partitionable( pub(crate) fn partition_ilp_analysis( ir: &mut [HydroRoot], op_id_to_parents: &HashMap>, + idbs: HashSet, decoupling_metadata: &RefCell, ) -> PartitionILPMetadata { // Make all cost expressions at all locations default to 0 @@ -908,14 +847,6 @@ pub(crate) fn partition_ilp_analysis( can_partition: HashMap::new(), }; - let inputs = all_inputs(ir, &decoupling_metadata.borrow().bottleneck); - // TODO: Known inaccuracy: This assumes that no edge will be created that converts EDBs to IDBs. That needs to be a constraint on decoupling - let idbs = nodes_dependent_on_inputs( - ir, - &decoupling_metadata.borrow().bottleneck, - &inputs, - op_id_to_parents, - ); let canonical_fields = create_canonical_fields( ir, &decoupling_metadata.borrow().bottleneck, diff --git a/hydro_optimize/src/partition_node_analysis.rs b/hydro_optimize/src/partition_node_analysis.rs index d442f02..4579d13 100644 --- a/hydro_optimize/src/partition_node_analysis.rs +++ b/hydro_optimize/src/partition_node_analysis.rs @@ -5,9 +5,8 @@ use hydro_lang::compile::ir::{HydroNode, HydroRoot, traverse_dfir}; use hydro_lang::location::dynamic::LocationId; use syn::visit::Visit; -use super::rewrites::{NetworkType, get_network_type}; use crate::partition_syn_analysis::{AnalyzeClosure, StructOrTuple, StructOrTupleIndex}; -use crate::rewrites::op_id_to_parents; +use crate::rewrites::{all_inputs, op_id_to_parents}; /// Create a mapping from all input nodes to their parents (across locations) fn all_inputs_parents( @@ -27,24 +26,6 @@ fn all_inputs_parents( .collect() } -/// Find all input nodes of a location -pub fn all_inputs(ir: &mut [HydroRoot], location: &LocationId) -> Vec { - let mut inputs = vec![]; - - traverse_dfir( - ir, - |_, _| {}, - |node, next_stmt_id| match get_network_type(node, location.root()) { - Some(NetworkType::Recv) | Some(NetworkType::SendRecv) => { - inputs.push(*next_stmt_id); - } - _ => {} - }, - ); - - inputs -} - pub struct InputDependencyMetadata { // Const fields pub location: LocationId, diff --git a/hydro_optimize/src/rewrites.rs b/hydro_optimize/src/rewrites.rs index ba1be46..5a28201 100644 --- a/hydro_optimize/src/rewrites.rs +++ b/hydro_optimize/src/rewrites.rs @@ -202,6 +202,82 @@ pub fn get_network_op_ids(ir: &mut [HydroRoot]) -> HashSet { network_ids } +/// Find all input nodes of a location +pub fn all_inputs(ir: &mut [HydroRoot], location: &LocationId) -> Vec { + let mut inputs = vec![]; + + traverse_dfir( + ir, + |_, _| {}, + |node, next_stmt_id| match get_network_type(node, location.root()) { + Some(NetworkType::Recv) | Some(NetworkType::SendRecv) => { + inputs.push(*next_stmt_id); + } + _ => {} + }, + ); + + inputs +} + +/// Add id to dependent_nodes if its parent_id is already in dependent_nodes +fn insert_dependent_node( + dependent_nodes: &RefCell>, + parent_id: Option, + id: usize, +) { + let mut dependent_nodes_borrow = dependent_nodes.borrow_mut(); + if let Some(parent_id) = parent_id + && dependent_nodes_borrow.contains(&parent_id) + { + dependent_nodes_borrow.insert(id); + } +} + +/// Returns IDs of nodes that are dependent on at least 1 input +pub fn nodes_dependent_on_inputs( + ir: &mut [HydroRoot], + location: &LocationId, + inputs: &[usize], + op_to_parents: &HashMap>, +) -> HashSet { + let dependent_nodes = RefCell::new(HashSet::from_iter(inputs.iter().cloned())); + let mut num_dependent_nodes = inputs.len(); + + loop { + traverse_dfir( + ir, + |root, next_stmt_id| { + if root.input_metadata().location_id.root() == location { + insert_dependent_node( + &dependent_nodes, + root.input_metadata().op.id, + *next_stmt_id, + ); + } + }, + |node, next_stmt_id| { + if node.metadata().location_id.root() == location.root() + && let Some(parents) = op_to_parents.get(next_stmt_id) + { + for parent_id in parents { + insert_dependent_node(&dependent_nodes, Some(*parent_id), *next_stmt_id); + } + } + }, + ); + + if dependent_nodes.borrow().len() == num_dependent_nodes { + // No new dependent nodes found, reached fixpoint + break; + } + num_dependent_nodes = dependent_nodes.borrow().len(); + } + + dependent_nodes.take() +} + + /// Check if the type is serializable. Currently a janky implementation that just looks for common unserializable types. /// Add to the list as new errors emerge. fn type_is_serializable(t: &DebugType) -> bool { From 7070bf99733f482d3356f0ff487a494eac9378b4 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 30 Jun 2026 19:30:48 +0000 Subject: [PATCH 22/29] Benchmarking works for protocols with multiple workloads, simplifications --- hydro_optimize/src/deploy_and_analyze.rs | 522 +++++++++--------- hydro_optimize/src/parse_results.rs | 111 +++- .../examples/benchmark_paxos.rs | 51 +- hydro_optimize_examples/examples/cas.rs | 76 +-- .../examples/network_calibrator.rs | 39 +- 5 files changed, 440 insertions(+), 359 deletions(-) diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index 4297fe9..6981196 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -14,13 +14,13 @@ use hydro_lang::prelude::{Cluster, FlowBuilder, Process}; use hydro_lang::telemetry::Sidecar; use tokio::sync::mpsc::UnboundedReceiver; -use crate::decouple_analysis::{IlpInputs, Rewrite, find_optimal_budget, project_total_cpu}; +use crate::decouple_analysis::{IlpInputs, Rewrite, find_optimal_budget}; use crate::deploy::{AWS_IO_TPS, AWS_NETWORK_BYTES_PER_SEC, HostType, ReusableHosts}; use crate::greedy_decouple_analysis::greedy_decouple_analysis; use crate::parse_results::{ OptimizationState, RawIlpInputs, RunMetadata, analyze_cluster_results, decoupled_cluster_name, - derive_per_op_load, find_bottleneck_from_run, find_latest_iteration, find_workload_dirs, - load_raw_ilp_inputs, max_throughput_for, original_cluster_name, + derive_per_op_load, find_bottleneck_across_runs, find_latest_iteration, find_workload_dirs, + get_csvs_in_dir, load_raw_ilp_inputs, max_throughput_for, original_cluster_name, }; use crate::reduce_pushdown::reduce_pushdown; use crate::reduce_pushdown_analysis::reduce_pushdown_decision; @@ -398,33 +398,33 @@ impl ReusableProcesses { /// Mutually exclusive optimization strategies that rewrite the IR before deployment. #[derive(Clone, Default)] -pub enum OptimizationKind { +pub enum Optimization { /// No IR rewriting, no counters. #[default] None, /// No IR rewriting, but insert counters to measure per-op cardinality. - CountersOnly, + Counters, /// Apply greedy decoupling, deploy decoupled system, gather per-operator SAR costs. /// Always inserts counters. BlowUpAnalysis, /// Insert size measuring nodes wherever decoupling is possible. SizeAnalysis, /// Deploy with `perf record` on non-excluded locations for flamegraph profiling. - PerfOnly, + Perf, /// Run ILP to find optimal decoupling, save Rewrite to file, then exit without deploying. /// Automatically loads applied rewrites from the latest opt dir. BottleneckElimination, } -impl OptimizationKind { +impl Optimization { pub fn label(&self) -> &'static str { match self { // Do not differentiate between None and BottleneckElimination, since both run the protocol without additional profiling - OptimizationKind::None | OptimizationKind::BottleneckElimination => "none", - OptimizationKind::CountersOnly => "counters", - OptimizationKind::BlowUpAnalysis => "blow_up", - OptimizationKind::SizeAnalysis => "size", - OptimizationKind::PerfOnly => "perf", + Optimization::None | Optimization::BottleneckElimination => "none", + Optimization::Counters => "counters", + Optimization::BlowUpAnalysis => "blow_up", + Optimization::SizeAnalysis => "size", + Optimization::Perf => "perf", } } } @@ -442,69 +442,40 @@ pub fn make_output_dir(name: &str, workload: &str, label: &str) -> std::path::Pa dir } +/// Per-workload compiled artifacts. These depend on the compiled program's location IDs, so +/// they are produced per workload by the `compile` closure rather than shared in +/// `BenchmarkConfig`. All workloads of a protocol must produce an identical IR topology (same +/// op ids / clusters), differing only in runtime data. #[derive(Clone, Default)] -pub struct Optimizations { - pub decoupling: bool, - pub partitioning: bool, - pub kind: OptimizationKind, +pub struct CompiledProgram { + pub clusters: ReusableClusters, + pub processes: ReusableProcesses, + /// Locations excluded from optimization (e.g. clients). Per-workload because it references + /// the compiled program's location IDs. pub exclude: HashSet, + pub location_id_to_cluster: HashMap, } -impl Optimizations { - pub fn with_decoupling(mut self) -> Self { - self.decoupling = true; - self - } - - pub fn with_partitioning(mut self) -> Self { - self.partitioning = true; - self - } - - /// Insert counters only (no IR rewriting). - pub fn with_counters_only(mut self) -> Self { - self.set_kind(OptimizationKind::CountersOnly); - self - } - - /// Insert size measuring nodes wherever decoupling is possible - pub fn with_size_analysis(mut self) -> Self { - self.set_kind(OptimizationKind::SizeAnalysis); - self - } - - /// Apply greedy decoupling, deploy decoupled system, gather per-operator SAR costs - pub fn with_blow_up_analysis(mut self) -> Self { - self.set_kind(OptimizationKind::BlowUpAnalysis); - self - } - - /// Deploy with `perf record` on non-excluded locations for flamegraph profiling. - pub fn with_perf_only(mut self) -> Self { - self.set_kind(OptimizationKind::PerfOnly); - self - } - - /// Run ILP-based bottleneck elimination (offline, no deployment). - /// Automatically finds the latest run directory for bottleneck detection - /// and loads applied rewrites from the latest opt dir. - pub fn with_bottleneck_elimination(mut self) -> Self { - self.set_kind(OptimizationKind::BottleneckElimination); - self +impl CompiledProgram { + pub fn new( + clusters: ReusableClusters, + processes: ReusableProcesses, + location_id_to_cluster: HashMap, + ) -> Self { + Self { + clusters, + processes, + exclude: HashSet::new(), + location_id_to_cluster, + } } + /// Excludes a location (cluster/process) from optimization: it won't be decoupled, + /// partitioned, profiled with `perf`, or chosen as a bottleneck. pub fn excluding(mut self, location: LocationId) -> Self { self.exclude.insert(location); self } - - fn set_kind(&mut self, kind: OptimizationKind) { - assert!( - matches!(self.kind, OptimizationKind::None), - "blow_up_analysis, size_analysis, and loaded rewrites are mutually exclusive" - ); - self.kind = kind; - } } /// `stability_second`: The second in which the protocol is expected to be stable, and its performance can be used as the basis for optimization. @@ -517,9 +488,8 @@ async fn deploy_and_analyze<'a>( reusable_hosts: &mut ReusableHosts, deployment: &mut Deployment, builder: BuiltFlow<'a>, - clusters: &ReusableClusters, - processes: &ReusableProcesses, - optimizations: &Optimizations, + program: &CompiledProgram, + kind: &Optimization, num_seconds: Option, ) -> RunMetadata { // Measure network (-n DEV) and CPU (-u) usage. -P 0 measures only core 0, where the @@ -530,12 +500,12 @@ async fn deploy_and_analyze<'a>( prefix: SAR_USAGE_PREFIX.to_string(), }; - let use_perf = matches!(optimizations.kind, OptimizationKind::PerfOnly); + let use_perf = matches!(kind, Optimization::Perf); // Insert all clusters & processes let mut deployable = builder.into_deploy(); - for (cluster_id, name, num_hosts) in clusters.named_clusters.iter() { - let excluded = optimizations.exclude.contains(cluster_id); + for (cluster_id, name, num_hosts) in program.clusters.named_clusters.iter() { + let excluded = program.exclude.contains(cluster_id); let hosts = reusable_hosts.get_cluster_hosts( deployment, name.clone(), @@ -544,12 +514,12 @@ async fn deploy_and_analyze<'a>( ); deployable = deployable.with_cluster_erased(cluster_id.key(), hosts); } - for (process_id, name) in processes.named_processes.iter() { - let excluded = optimizations.exclude.contains(process_id); + for (process_id, name) in program.processes.named_processes.iter() { + let excluded = program.exclude.contains(process_id); let host = reusable_hosts.get_process_host(deployment, name.clone(), use_perf && !excluded); deployable = deployable.with_process_erased(process_id.key(), host); } - for excluded_location in optimizations.exclude.iter() { + for excluded_location in program.exclude.iter() { deployable = deployable.set_batch_limit_from(excluded_location.key(), BATCH_PULL_LIMIT); } deployable = deployable.with_sidecar_all(&sar_sidecar); @@ -587,15 +557,15 @@ pub struct BenchmarkArgs { pub aws: bool, } +/// Static benchmark configuration, shared across all workloads of a protocol. Per-workload, +/// program-dependent data (clusters, processes, excluded ids, location map) lives in +/// `CompiledProgram`; the workload name comes from the `(params, name)` workload mapping. #[derive(Clone)] pub struct BenchmarkConfig { + /// Base protocol name (e.g. "CAS"). Output dirs are `{name}[_optN]_{workload}_{label}`. pub name: String, - /// Workload variant (e.g. "read_heavy"). Determines output subdirectory name. - pub workload: String, - pub clusters: ReusableClusters, - pub processes: ReusableProcesses, - pub optimizations: Optimizations, - pub location_id_to_cluster: HashMap, + /// Which optimization/analysis strategy to run. Static across workloads. + pub kind: Optimization, pub num_physical_clients: usize, pub start_virtual_clients: usize, pub virtual_clients_step: usize, @@ -764,12 +734,15 @@ async fn run_scaling_loop<'a>( deployment: &mut Deployment, built: &BuiltFlow<'a>, config: &BenchmarkConfig, - optimizations: &Optimizations, + program: &CompiledProgram, + run_name: &str, + workload: &str, + kind: &Optimization, location_to_original_ops: &HashMap>, pre_rewrite_parents: Option<&HashMap>>, size_analysis_ops: &HashSet, ) -> RunMetadata { - let output_dir = make_output_dir(&config.name, &config.workload, optimizations.kind.label()); + let output_dir = make_output_dir(run_name, workload, kind.label()); let ir = built.ir(); save_id(&mut deep_clone(ir), &output_dir.join("id.txt")); let mut final_run_metadata = RunMetadata::default(); @@ -785,8 +758,8 @@ async fn run_scaling_loop<'a>( while successful_runs < config.num_runs { println!( "Running {} ({}) with {} virtual clients (run {})", - config.name, - optimizations.kind.label(), + run_name, + kind.label(), num_virtual, run ); @@ -800,9 +773,8 @@ async fn run_scaling_loop<'a>( reusable_hosts, deployment, finalized, - &config.clusters, - &config.processes, - optimizations, + program, + kind, Some(RUN_SECONDS), ) .await; @@ -813,16 +785,16 @@ async fn run_scaling_loop<'a>( "Run throughput: {}, num_clients: {}", run_throughput, num_virtual ); - run_metadata.print_run_summary(&config.location_id_to_cluster, MEASUREMENT_SECOND); + run_metadata.print_run_summary(&program.location_id_to_cluster, MEASUREMENT_SECOND); run_metadata.save_run_metadata( - &config.location_id_to_cluster, + &program.location_id_to_cluster, &output_dir, config.num_physical_clients, num_virtual, run, ); - if matches!(optimizations.kind, OptimizationKind::BlowUpAnalysis) { + if matches!(kind, Optimization::BlowUpAnalysis) { run_metadata.save_blow_up_stats( &output_dir, &mut deep_clone(ir), @@ -835,7 +807,7 @@ async fn run_scaling_loop<'a>( if !size_analysis_ops.is_empty() { run_metadata.save_size_analysis(&output_dir, size_analysis_ops); } - if matches!(optimizations.kind, OptimizationKind::CountersOnly) { + if matches!(kind, Optimization::Counters) { run_metadata.save_counters( &output_dir, config.num_physical_clients, @@ -865,11 +837,13 @@ async fn run_scaling_loop<'a>( } else { throughput_sum += run_throughput; successful_runs += 1; + // TODO: Remove after debugging CAS + // break 'outer; } run += 1; final_run_metadata = run_metadata; - if matches!(optimizations.kind, OptimizationKind::SizeAnalysis) + if matches!(kind, Optimization::SizeAnalysis) || config.calibrate_message_sizes.is_some() { break 'outer; @@ -908,12 +882,16 @@ async fn run_scaling_loop<'a>( /// Runs a single analysis pass (counters, size, blow_up, or perf) on the given built flow. /// Deploys with the appropriate instrumentation, scales virtual clients, and saves results. +#[allow(clippy::too_many_arguments)] async fn run_analysis_pass<'a>( reusable_hosts: &mut ReusableHosts, deployment: &mut Deployment, built: &BuiltFlow<'a>, - config: &mut BenchmarkConfig, - kind: OptimizationKind, + config: &BenchmarkConfig, + program: &mut CompiledProgram, + run_name: &str, + workload: &str, + kind: Optimization, ) { println!("=== Running {} analysis ===", kind.label()); @@ -926,25 +904,23 @@ async fn run_analysis_pass<'a>( let analysis_built = builder.finalize(); let analysis_built = match &kind { - OptimizationKind::CountersOnly => { - insert_counters(analysis_built, &config.optimizations.exclude) - } - OptimizationKind::BlowUpAnalysis => { + Optimization::Counters => insert_counters(analysis_built, &program.exclude), + Optimization::BlowUpAnalysis => { let (built, loc_ops, parents) = apply_blow_up_analysis( analysis_built, - &mut config.clusters, - &config.optimizations.exclude, - &mut config.location_id_to_cluster, + &mut program.clusters, + &program.exclude, + &mut program.location_id_to_cluster, ); pre_rewrite_parents = Some(parents); location_to_original_ops = loc_ops; built } - OptimizationKind::SizeAnalysis => { + Optimization::SizeAnalysis => { let mut captured_ops = HashSet::new(); let built = analysis_built.optimize_with(|leaf| { let per_loc_rewrites = - greedy_decouple_analysis(leaf, &config.optimizations.exclude, &config.clusters); + greedy_decouple_analysis(leaf, &program.exclude, &program.clusters); let network_ops: HashSet = per_loc_rewrites .iter() .flat_map(|r| r.op_to_network.keys().copied()) @@ -957,14 +933,8 @@ async fn run_analysis_pass<'a>( size_analysis_ops = captured_ops; built } - OptimizationKind::PerfOnly | OptimizationKind::None => analysis_built, - OptimizationKind::BottleneckElimination => unreachable!(), - }; - - let optimizations = Optimizations { - kind: kind.clone(), - exclude: config.optimizations.exclude.clone(), - ..Default::default() + Optimization::Perf | Optimization::None => analysis_built, + Optimization::BottleneckElimination => unreachable!(), }; run_scaling_loop( @@ -972,7 +942,10 @@ async fn run_analysis_pass<'a>( deployment, &analysis_built, config, - &optimizations, + program, + run_name, + workload, + &kind, &location_to_original_ops, pre_rewrite_parents.as_ref(), &size_analysis_ops, @@ -982,26 +955,77 @@ async fn run_analysis_pass<'a>( println!("=== {} analysis complete ===", kind.label()); } -/// Runs the ILP-based bottleneck elimination workflow. +/// Compiles and finalizes a workload's program (inject ids + reduce pushdown), returning the +/// optimized base `BuiltFlow` and its `CompiledProgram`. +fn build_workload<'a, W, F>(compile: &F, workload: &W) -> (BuiltFlow<'a>, CompiledProgram) +where + F: Fn(&W) -> (FlowBuilder<'a>, CompiledProgram), +{ + let (builder, program) = compile(workload); + let built = builder.finalize().optimize_with(|leaf| { + inject_id(leaf); + let decision = reduce_pushdown_decision(leaf, &program.exclude); + reduce_pushdown(leaf, decision); + }); + (built, program) +} + +/// Applies the rewrites currently recorded in `state` to a fresh copy of `base_built`, +/// updating `program`'s clusters/loc-map with any new decoupled clusters. Returns `None` if +/// there are no prior rewrites to apply (so callers fall back to `base_built`). +fn apply_prior_rewrites<'a>( + base_built: &BuiltFlow<'a>, + state: &OptimizationState, + program: &mut CompiledProgram, +) -> Option> { + if state.applied.is_empty() { + return None; + } + let rewrites = state.rewrites_to_apply(); + let mut builder = FlowBuilder::from_built(base_built); + builder.replace_ir(deep_clone(base_built.ir())); + Some(apply_loaded_rewrites( + builder.finalize(), + &mut program.clusters, + &rewrites, + &mut program.location_id_to_cluster, + )) +} + +/// Plans the next optimization iteration across all workloads and returns `(run_name, rewrites)` +/// for the caller to benchmark. Workloads share an identical IR (they differ only in runtime +/// data), so stats are op-id-keyed and merged with element-wise maxes across workloads. +/// Exits the process when optimization is complete (exhausted / no improvement). /// -/// 1. Apply prior rewrites from state -/// 2. Find bottleneck from latest `_none` run -/// 3. If no cached ILP rewrites for bottleneck → run analyses, compute ILP -/// 4. Apply next budget for bottleneck cluster -async fn run_bottleneck_elimination<'a>( +/// Phases: +/// 1. First-ever run: no rewrites yet — return the base name and an empty rewrite set. +/// 2. Find the bottleneck across all workloads' latest `_none` runs (max saturation per cluster). +/// 3. If no cached rewrites: run perf/counters/size analyses for *each* workload (gated per +/// workload on disk), then compute the ILP ONCE on the max-merged stats. +/// 4. Apply the next budget once and return its rewrites. +async fn prepare_bottleneck_rewrites<'a, W, F>( reusable_hosts: &mut ReusableHosts, deployment: &mut Deployment, - built: BuiltFlow<'a>, - config: &mut BenchmarkConfig, -) -> BuiltFlow<'a> { + config: &BenchmarkConfig, + workloads: &[(W, String)], + compile: &F, +) -> (String, Vec) +where + F: Fn(&W) -> (FlowBuilder<'a>, CompiledProgram), +{ + assert!( + !workloads.is_empty(), + "No workloads provided for optimization" + ); + let base = Path::new(CALIBRATION_DIR); let base_name = config.name.clone(); let state_path = base.join(format!("{}_optimization_state.json", base_name)); let mut state = OptimizationState::load(&state_path); - // Check if a previous _none run exists + // === Phase 1: first-ever run — no rewrites yet; benchmark the base program. === let Some(latest_iter) = find_latest_iteration(base, &base_name) else { - return built; // First ever run — just benchmark the base + return (base_name, Vec::new()); }; let iteration = latest_iter + 1; let prev_name = if latest_iter == 0 { @@ -1010,7 +1034,7 @@ async fn run_bottleneck_elimination<'a>( format!("{}_opt{}", base_name, latest_iter) }; - // Early exit if last optimization didn't improve throughput + // Early exit if last optimization didn't improve throughput (max across workloads). if latest_iter >= 2 { let thr_prev = max_throughput_for(base, &prev_name); let thr_prev_prev = @@ -1031,18 +1055,24 @@ async fn run_bottleneck_elimination<'a>( } } - let excluded_names: HashSet = config - .optimizations + // === Phase 2: find the bottleneck across all workloads' latest `_none` runs. === + // Excluded ids depend on the compiled program, so build one workload to resolve names. + // (Use `build_workload` rather than `compile` directly: an unfinalized `FlowBuilder` + // panics on drop, so the builder must be finalized even though we discard the result.) + let (_first_built, first_program) = build_workload(compile, &workloads[0].0); + let excluded_names: HashSet = first_program .exclude .iter() - .filter_map(|loc| config.location_id_to_cluster.get(loc).cloned()) + .filter_map(|loc| first_program.location_id_to_cluster.get(loc).cloned()) .collect(); - let prev_run_dir = find_workload_dirs(base, &prev_name, OptimizationKind::None.label()) - .into_iter() - .next() - .unwrap_or_else(|| panic!("No _none run found for '{}'", prev_name)); - let bottleneck_name = find_bottleneck_from_run( - &prev_run_dir, + let prev_run_dirs = find_workload_dirs(base, &prev_name, Optimization::None.label()); + assert!( + !prev_run_dirs.is_empty(), + "No _none run found for '{}'", + prev_name + ); + let bottleneck_name = find_bottleneck_across_runs( + &prev_run_dirs, AWS_NETWORK_BYTES_PER_SEC, AWS_IO_TPS, &excluded_names, @@ -1057,58 +1087,66 @@ async fn run_bottleneck_elimination<'a>( std::process::exit(0); } - // If no cached rewrites for this cluster, run analyses and compute ILP + // === Phase 3: if no cached rewrites, analyze every workload then compute the ILP once. === if !state.cluster_rewrites.contains_key(&bottleneck_cluster) { - let analysis_name; - let analysis_built; - // Snapshot the base clusters/loc-map. When prior rewrites are replayed below, - // they create new (decoupled/partitioned) clusters that the analysis passes must - // deploy. Those passes deploy with `config.clusters`/`config.location_id_to_cluster`, - // so we apply the rewrites directly into `config` here, then restore the base - // snapshots before the final apply recreates them from scratch. - let base_clusters = config.clusters.clone(); - let base_loc_map = config.location_id_to_cluster.clone(); - if state.applied.is_empty() { - analysis_name = base_name.clone(); - analysis_built = None; + let analysis_name = if state.applied.is_empty() { + base_name.clone() } else { - let name = format!("{}_opt{}", base_name, iteration); - config.name = name.clone(); - analysis_name = name; - - let rewrites = state.rewrites_to_apply(); - let mut builder = FlowBuilder::from_built(&built); - builder.replace_ir(deep_clone(built.ir())); - analysis_built = Some(apply_loaded_rewrites( - builder.finalize(), - &mut config.clusters, - &rewrites, - &mut config.location_id_to_cluster, - )); - } + format!("{}_opt{}", base_name, iteration) + }; - for kind in &[ - OptimizationKind::PerfOnly, - OptimizationKind::CountersOnly, - OptimizationKind::SizeAnalysis, - // OptimizationKind::BlowUpAnalysis, - // TODO: Reenable blow up analysis once vcpu limit has been addressed - ] { - if find_workload_dirs(base, &analysis_name, kind.label()).is_empty() { - let run_built = analysis_built.as_ref().unwrap_or(&built); - run_analysis_pass(reusable_hosts, deployment, run_built, config, kind.clone()) - .await; + // Run perf/counters/size for each workload, gated on that workload's own dir so every + // workload gets analyzed (not just the first). + for (params, workload) in workloads { + let (base_built, mut program) = build_workload(compile, params); + let analysis_built = apply_prior_rewrites(&base_built, &state, &mut program); + let run_built = analysis_built.as_ref().unwrap_or(&base_built); + for kind in &[ + Optimization::Perf, + Optimization::Counters, + Optimization::SizeAnalysis, + ] { + // Skip if this workload's analysis already completed. Build the path directly — + // do NOT use `make_output_dir`, which deletes and recreates the directory. Gate + // on CSV presence so a previously-created-but-empty dir is re-run. + let dir = Path::new(CALIBRATION_DIR).join(format!( + "{}_{}_{}", + analysis_name, + workload, + kind.label() + )); + if dir.is_dir() && !get_csvs_in_dir(&dir).is_empty() { + continue; + } + run_analysis_pass( + reusable_hosts, + deployment, + run_built, + config, + &mut program, + &analysis_name, + workload, + kind.clone(), + ) + .await; } } + // Barrier: every workload is analyzed. Merge stats (max across workloads) and solve once. let raw_ilp_inputs = load_raw_ilp_inputs(base, &analysis_name); - let bottleneck_loc = config + + // Canonical IR for the ILP: any workload's program (they share structure), with prior + // rewrites applied so the op ids and locations match the analyzed program. + let (canon_base, mut canon_program) = build_workload(compile, &workloads[0].0); + let canon_built = + apply_prior_rewrites(&canon_base, &state, &mut canon_program).unwrap_or(canon_base); + let bottleneck_loc = canon_program .location_id_to_cluster .iter() .find(|(_, n)| *n == &bottleneck_name) .map(|(id, _)| id.clone()) .unwrap_or_else(|| panic!("Bottleneck cluster '{}' not found", bottleneck_name)); - let cluster_size = config + let cluster_size = canon_program .clusters .location_name_and_num(&bottleneck_loc) .map(|(_, n)| n) @@ -1122,11 +1160,7 @@ async fn run_bottleneck_elimination<'a>( network_cost_table, } = raw_ilp_inputs; - let mut ir = if let Some(ref ab) = analysis_built { - deep_clone(ab.ir()) - } else { - deep_clone(built.ir()) - }; + let mut ir = deep_clone(canon_built.ir()); let network_op_ids = get_network_op_ids(&mut ir); let cycles = cycle_source_to_sink_parent(&mut ir); let op_parents = op_id_to_parents(&mut ir, Some(&bottleneck_loc), &cycles); @@ -1146,37 +1180,37 @@ async fn run_bottleneck_elimination<'a>( state .cluster_rewrites .insert(bottleneck_cluster.clone(), computed_rewrites); - - // Restore the base clusters/loc-map: the final apply below recreates the - // decoupled clusters from the base program, so `config` must not already - // contain the analysis-pass copies (which would duplicate them). - config.clusters = base_clusters; - config.location_id_to_cluster = base_loc_map; } - // Apply next budget for the bottleneck cluster + // === Phase 4: apply the next budget once and return its rewrites for the caller to run. === let budget = state.next_budget(&bottleneck_cluster); println!( "Bottleneck '{}': applying budget {}.", bottleneck_cluster, budget ); state.applied.insert(bottleneck_cluster, budget); - let built = apply_loaded_rewrites( - built, - &mut config.clusters, - &state.rewrites_to_apply(), - &mut config.location_id_to_cluster, - ); - state.save(&state_path); - config.name = format!("{}_opt{}", base_name, iteration); - built + + let opt_name = format!("{}_opt{}", base_name, iteration); + (opt_name, state.rewrites_to_apply()) } -pub async fn benchmark_protocol<'a>( +/// Runs the benchmark/optimization flow over one or more workloads. Each workload is built by +/// `compile(¶ms)` and must produce an identical IR structure (op-id topology), differing +/// only in runtime data (e.g. write ratio), since stats are merged across workloads by op id. +/// Each workload's name (the second tuple element) must be a unique, stable label — it +/// discriminates output directories on disk. +pub async fn benchmark_protocol<'a, W>( args: BenchmarkArgs, - run_benchmark: impl Fn() -> (FlowBuilder<'a>, BenchmarkConfig), + config: BenchmarkConfig, + workloads: &[(W, String)], + compile: impl Fn(&W) -> (FlowBuilder<'a>, CompiledProgram), ) -> (Vec, RunMetadata) { + assert!(!workloads.is_empty(), "Must provide at least one workload"); + assert!( + config.num_runs > 0, + "Must run at least one iteration of the benchmark" + ); let mut deployment = Deployment::new(); let host_type: HostType = if let Some(project) = args.gcp.clone() { HostType::Gcp { project } @@ -1187,69 +1221,65 @@ pub async fn benchmark_protocol<'a>( }; let mut reusable_hosts = ReusableHosts::new(&host_type); - let (builder, mut config) = run_benchmark(); - assert!( - config.num_runs > 0, - "Must run at least one iteration of the benchmark" - ); - if config.calibrate_message_sizes.is_some() { reusable_hosts.set_single_host(true); } - // Apply the selected optimization strategy exactly once. - let exclude = config.optimizations.exclude.clone(); - let built = builder.finalize().optimize_with(|leaf| { - inject_id(leaf); - let decision = reduce_pushdown_decision(leaf, &exclude); - reduce_pushdown(leaf, decision); - }); - - let built = match &config.optimizations.kind { - OptimizationKind::None => built, - OptimizationKind::BottleneckElimination => { - run_bottleneck_elimination(&mut reusable_hosts, &mut deployment, built, &mut config) - .await - } - _ => { - println!("Warning: Explicit optimization kinds are no longer allowed as parameters"); - built - } + // Decide what to benchmark: bottleneck elimination first runs the analyses + ILP and returns + // the rewrites + run name; every other mode benchmarks the base program (no rewrites). + let (run_name, rewrites) = if matches!(config.kind, Optimization::BottleneckElimination) { + prepare_bottleneck_rewrites( + &mut reusable_hosts, + &mut deployment, + &config, + workloads, + &compile, + ) + .await + } else { + (config.name.clone(), Vec::new()) }; - // Run a loop for each network calibration size, if provided - let final_run_metadata = if let Some(ref sizes) = config.calibrate_message_sizes { - let mut last_metadata = RunMetadata::default(); - for &size in sizes { - reusable_hosts.insert_env(HYDRO_NET_CALIBRATE_ENV.to_string(), size.to_string()); - let mut iter_config = config.clone(); - iter_config.name = format!("{}_{}b", config.name, size); - last_metadata = run_scaling_loop( + // Benchmark the (possibly rewritten) program for each workload, with optional calibration. + let mut last = (Vec::new(), RunMetadata::default()); + for (params, workload) in workloads { + let (base_built, mut program) = build_workload(&compile, params); + let built = apply_loaded_rewrites( + base_built, + &mut program.clusters, + &rewrites, + &mut program.location_id_to_cluster, + ); + // One deploy per calibration size (under a `{run_name}_{size}b` name), or a single + // deploy under `run_name` when not calibrating. + let scaling_runs: Vec<(String, Option)> = match &config.calibrate_message_sizes { + Some(sizes) => sizes + .iter() + .map(|&size| (format!("{}_{}b", run_name, size), Some(size))) + .collect(), + None => vec![(run_name.clone(), None)], + }; + let mut meta = RunMetadata::default(); + for (scaling_name, calibrate_size) in &scaling_runs { + if let Some(size) = calibrate_size { + reusable_hosts.insert_env(HYDRO_NET_CALIBRATE_ENV.to_string(), size.to_string()); + } + meta = run_scaling_loop( &mut reusable_hosts, &mut deployment, &built, - &iter_config, - &config.optimizations, + &config, + &program, + scaling_name, + workload, + &config.kind, &HashMap::new(), None, &HashSet::new(), ) .await; } - last_metadata - } else { - run_scaling_loop( - &mut reusable_hosts, - &mut deployment, - &built, - &config, - &config.optimizations, - &HashMap::new(), - None, - &HashSet::new(), - ) - .await - }; - - (deep_clone(built.ir()), final_run_metadata) + last = (deep_clone(built.ir()), meta); + } + last } diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index bd4918d..b4f6f43 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -15,7 +15,7 @@ use tokio::sync::mpsc::UnboundedReceiver; use crate::decouple_analysis::Rewrite; use crate::deploy_and_analyze::{ - MEASUREMENT_SECOND, MetricLogs, OptimizationKind, START_MEASUREMENT_SECOND, + MEASUREMENT_SECOND, MetricLogs, Optimization, START_MEASUREMENT_SECOND, inject_inferred_counters, }; @@ -23,12 +23,13 @@ use crate::deploy_and_analyze::{ /// 1. Finds the run suffix (e.g. "10c_500vc_r0") with highest avg throughput. /// 2. For that run, reads all cluster CSVs and returns the cluster name whose SAR stats /// are closest to saturation (100% CPU, 100% memory, or at/above max network/IO). -pub fn find_bottleneck_from_run( +/// For one run directory, returns each cluster's saturation (max resource utilization) at the +/// highest-throughput run suffix. Returns an empty map if the directory has no CSVs. +fn cluster_saturations_from_run( run_dir: &Path, network_bytes_per_sec: f64, io_tps: f64, - excluded_names: &HashSet, -) -> String { +) -> HashMap { let csv_re = Regex::new(r"^(.+)_(\d+c_\d+vc_r\d+)\.csv$").unwrap(); // Group CSVs by suffix @@ -49,7 +50,7 @@ pub fn find_bottleneck_from_run( } // Find suffix with highest throughput (use first CSV that has throughput data) - let best_suffix = suffix_to_files + let Some(best_suffix) = suffix_to_files .keys() .max_by_key(|suffix| { suffix_to_files[*suffix] @@ -58,8 +59,10 @@ pub fn find_bottleneck_from_run( .max() .unwrap_or(0) }) - .expect("No CSVs found in run dir") - .clone(); + .cloned() + else { + return HashMap::new(); + }; // For the best suffix, score each cluster by saturation let score = |s: &SarStats| -> f64 { @@ -77,22 +80,76 @@ pub fn find_bottleneck_from_run( .fold(0.0_f64, f64::max) }; - // Excluded clusters/processes (e.g. clients) are never optimization targets, so they - // are not eligible to be the bottleneck. Compare against the base cluster name so that - // decoupled/partitioned variants (e.g. "client_loc1") are excluded too. - let (bottleneck_name, _) = suffix_to_files[&best_suffix] + let mut saturations: HashMap = HashMap::new(); + for (cluster_name, path) in &suffix_to_files[&best_suffix] { + let s = score(&csv_avg_sar(path)); + let entry = saturations.entry(cluster_name.clone()).or_insert(f64::MIN); + if s > *entry { + *entry = s; + } + } + saturations +} + +/// Returns the most-saturated cluster name in `saturations`, excluding any whose base name is +/// in `excluded_names` (so decoupled/partitioned variants like "client_loc1" are excluded too). +fn most_saturated_cluster( + saturations: &HashMap, + excluded_names: &HashSet, +) -> String { + saturations .iter() - .filter(|(cluster_name, _)| !excluded_names.contains(original_cluster_name(cluster_name))) - .map(|(cluster_name, path)| { - let avg_sar = csv_avg_sar(path); - (cluster_name.clone(), score(&avg_sar)) - }) + .filter(|(name, _)| !excluded_names.contains(original_cluster_name(name))) .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap()) - .expect("No non-excluded cluster CSVs for best suffix"); + .map(|(name, _)| name.clone()) + .expect("No non-excluded cluster found in run") +} +/// Finds the bottleneck cluster from a single run folder's CSV data. +/// 1. Finds the run suffix (e.g. "10c_500vc_r0") with highest avg throughput. +/// 2. For that run, returns the (non-excluded) cluster name whose SAR stats are closest to +/// saturation (100% CPU, 100% memory, or at/above max network/IO). +pub fn find_bottleneck_from_run( + run_dir: &Path, + network_bytes_per_sec: f64, + io_tps: f64, + excluded_names: &HashSet, +) -> String { + let saturations = cluster_saturations_from_run(run_dir, network_bytes_per_sec, io_tps); + let bottleneck_name = most_saturated_cluster(&saturations, excluded_names); + println!("Bottleneck from run {}: {}", run_dir.display(), bottleneck_name); + bottleneck_name +} + +/// Finds the bottleneck cluster across multiple workloads' run folders, taking the max +/// saturation per cluster across all of them. This picks the cluster that is the worst +/// bottleneck in *any* workload, so the optimizer targets the cluster that limits the +/// hardest workload rather than an arbitrary single one. +pub fn find_bottleneck_across_runs( + run_dirs: &[PathBuf], + network_bytes_per_sec: f64, + io_tps: f64, + excluded_names: &HashSet, +) -> String { + let mut merged: HashMap = HashMap::new(); + for dir in run_dirs { + for (name, s) in cluster_saturations_from_run(dir, network_bytes_per_sec, io_tps) { + let entry = merged.entry(name).or_insert(f64::MIN); + if s > *entry { + *entry = s; + } + } + } + assert!( + !merged.is_empty(), + "No cluster CSVs found across {} run dir(s)", + run_dirs.len() + ); + let bottleneck_name = most_saturated_cluster(&merged, excluded_names); println!( - "Bottleneck from run: {} (suffix: {})", - bottleneck_name, best_suffix + "Bottleneck across {} workload(s): {}", + run_dirs.len(), + bottleneck_name ); bottleneck_name } @@ -159,7 +216,7 @@ pub fn csv_avg_throughput(path: &Path) -> usize { /// Returns the max average throughput across all CSVs in `_none` workload dirs for `name`. pub fn max_throughput_for(base_dir: &Path, name: &str) -> usize { - find_workload_dirs(base_dir, name, OptimizationKind::None.label()) + find_workload_dirs(base_dir, name, Optimization::None.label()) .iter() .flat_map(|d| get_csvs_in_dir(d)) .map(|e| csv_avg_throughput(&e.path())) @@ -238,10 +295,10 @@ impl OptimizationState { /// Loads ILP inputs by scanning all workload directories for a given protocol name and label, /// taking the element-wise max across workloads. pub fn load_raw_ilp_inputs(base_dir: &Path, name: &str) -> RawIlpInputs { - let blow_up_dirs = find_workload_dirs(base_dir, name, OptimizationKind::BlowUpAnalysis.label()); - let size_dirs = find_workload_dirs(base_dir, name, OptimizationKind::SizeAnalysis.label()); - let counters_dirs = find_workload_dirs(base_dir, name, OptimizationKind::CountersOnly.label()); - let perf_dirs = find_workload_dirs(base_dir, name, OptimizationKind::PerfOnly.label()); + let blow_up_dirs = find_workload_dirs(base_dir, name, Optimization::BlowUpAnalysis.label()); + let size_dirs = find_workload_dirs(base_dir, name, Optimization::SizeAnalysis.label()); + let counters_dirs = find_workload_dirs(base_dir, name, Optimization::Counters.label()); + let perf_dirs = find_workload_dirs(base_dir, name, Optimization::Perf.label()); assert!(!size_dirs.is_empty(), "No size dirs for '{}'", name); assert!(!counters_dirs.is_empty(), "No counters dirs for '{}'", name); @@ -391,7 +448,7 @@ pub fn load_calibration_table(calibration_dir: &Path) -> NetworkCostTable { /// Finds the bottleneck run directory: the perf dir with highest throughput across workloads. pub fn find_bottleneck_run_dir(base_dir: &Path, name: &str) -> PathBuf { - let perf_label = OptimizationKind::PerfOnly.label(); + let perf_label = Optimization::Perf.label(); let perf_dirs = find_workload_dirs(base_dir, name, perf_label); assert!(!perf_dirs.is_empty(), "No perf dirs for '{}'", name); // Use the one with highest throughput CSVs @@ -474,13 +531,13 @@ pub fn decoupled_cluster_name(original_name: &str, loc_idx: usize) -> String { /// Returns `None` if no `_none` run has completed, `Some(0)` if only `{name}_default_none` /// exists, `Some(N)` if `{name}_optN_default_none` is the highest. pub fn find_latest_iteration(base_dir: &Path, name: &str) -> Option { - let base_none = find_workload_dirs(base_dir, name, OptimizationKind::None.label()); + let base_none = find_workload_dirs(base_dir, name, Optimization::None.label()); if base_none.is_empty() { return None; } let prefix = format!("{}_opt", name); - let suffix = format!("_{}", OptimizationKind::None.label()); + let suffix = format!("_{}", Optimization::None.label()); let max_opt = std::fs::read_dir(base_dir) .into_iter() .flatten() diff --git a/hydro_optimize_examples/examples/benchmark_paxos.rs b/hydro_optimize_examples/examples/benchmark_paxos.rs index 0abb432..09902f2 100644 --- a/hydro_optimize_examples/examples/benchmark_paxos.rs +++ b/hydro_optimize_examples/examples/benchmark_paxos.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use clap::{ArgAction, Parser}; use hydro_lang::location::Location; use hydro_optimize::deploy_and_analyze::{ - BenchmarkArgs, BenchmarkConfig, NUM_PHYSICAL_CLIENTS, Optimizations, ReusableClusters, - ReusableProcesses, benchmark_protocol, + BenchmarkArgs, BenchmarkConfig, CompiledProgram, NUM_PHYSICAL_CLIENTS, Optimization, + ReusableClusters, ReusableProcesses, benchmark_protocol, }; use hydro_optimize_examples::print_parseable_bench_results; use hydro_test::cluster::paxos::{CorePaxos, PaxosConfig}; @@ -33,20 +33,35 @@ struct Args { #[tokio::main] async fn main() { let args = Args::parse(); - let optimize = args.optimize; + + let config = BenchmarkConfig { + name: "Paxos".to_string(), + kind: if args.optimize { + Optimization::BottleneckElimination + } else { + Optimization::None + }, + num_physical_clients: NUM_PHYSICAL_CLIENTS, + start_virtual_clients: 11, + virtual_clients_step: 10, + num_runs: 1, + calibrate_message_sizes: None, + }; let (_ir, _final_run_metadata) = benchmark_protocol( BenchmarkArgs { gcp: args.gcp, aws: args.aws, }, - move || { + config, + &[((), "default".to_string())], + move |_: &()| { let num_clients = NUM_PHYSICAL_CLIENTS; let f = 1; let checkpoint_frequency = 1000; - let i_am_leader_send_timeout = 5; - let i_am_leader_check_timeout = 10; - let i_am_leader_check_timeout_delay_multiplier = 15; + let i_am_leader_send_timeout = 1; + let i_am_leader_check_timeout = 30; + let i_am_leader_check_timeout_delay_multiplier = 35; let print_result_frequency = 1000; let mut builder = hydro_lang::compile::builder::FlowBuilder::new(); @@ -100,29 +115,11 @@ async fn main() { .with_cluster(replicas, f + 1); let processes = ReusableProcesses::default().with_process(client_aggregator); - let mut optimizations = Optimizations::default() + let program = CompiledProgram::new(clusters, processes, location_id_to_cluster) .excluding(client_id.clone()) .excluding(client_aggregator_id); - if optimize { - optimizations = optimizations.with_bottleneck_elimination(); - } - ( - builder, - BenchmarkConfig { - name: "Paxos".to_string(), - workload: "default".to_string(), - clusters, - processes, - optimizations, - location_id_to_cluster, - num_physical_clients: num_clients, - start_virtual_clients: 1, - virtual_clients_step: 10, - num_runs: 1, - calibrate_message_sizes: None, - }, - ) + (builder, program) }, ) .await; diff --git a/hydro_optimize_examples/examples/cas.rs b/hydro_optimize_examples/examples/cas.rs index 9b3077f..52968cb 100644 --- a/hydro_optimize_examples/examples/cas.rs +++ b/hydro_optimize_examples/examples/cas.rs @@ -1,11 +1,10 @@ use std::collections::HashMap; use clap::{ArgAction, Parser}; -use hydro_lang::viz::config::GraphConfig; use hydro_lang::{location::Location, prelude::FlowBuilder}; use hydro_optimize::deploy_and_analyze::{ - BenchmarkArgs, BenchmarkConfig, NUM_PHYSICAL_CLIENTS, Optimizations, ReusableClusters, - ReusableProcesses, VIRTUAL_CLIENTS_STEP, benchmark_protocol, + BenchmarkArgs, BenchmarkConfig, CompiledProgram, NUM_PHYSICAL_CLIENTS, Optimization, + ReusableClusters, ReusableProcesses, benchmark_protocol, }; use hydro_optimize_examples::compare_and_swap::cas_bench::cas_bench; @@ -16,9 +15,6 @@ use hydro_optimize_examples::compare_and_swap::cas_bench::cas_bench; .multiple(false) ))] struct Args { - #[command(flatten)] - graph: GraphConfig, - /// Use Gcp for deployment (provide project name) #[arg(long)] gcp: Option, @@ -32,9 +28,12 @@ struct Args { optimize: bool, } -const WRITE_RATIOS: &[usize] = &[10, 100]; +const WRITE_RATIOS: &[usize] = &[0, 100]; +// const WRITE_RATIOS: &[usize] = &[100, 0]; -fn run_benchmark<'a>(write_ratio: usize, optimize: bool) -> (FlowBuilder<'a>, BenchmarkConfig) { +/// Compiles the CAS program for a given write ratio. All write ratios produce an identical IR +/// (the ratio is a runtime constant inside the client), so they share one optimization run. +fn compile<'a>(write_ratio: usize) -> (FlowBuilder<'a>, CompiledProgram) { let f = 1; let retry_timeout = 5000; let print_result_frequency = 1000; @@ -67,42 +66,43 @@ fn run_benchmark<'a>(write_ratio: usize, optimize: bool) -> (FlowBuilder<'a>, Be .with_cluster(clients, num_clients); let processes = ReusableProcesses::default().with_process(client_aggregator); - let mut optimizations = Optimizations::default() + let program = CompiledProgram::new(clusters, processes, location_id_to_cluster) .excluding(client_id) .excluding(client_aggregator_id); - if optimize { - optimizations = optimizations.with_bottleneck_elimination(); - } - ( - builder, - BenchmarkConfig { - name: "CAS".to_string(), - workload: format!("write{write_ratio}"), - clusters, - processes, - optimizations, - location_id_to_cluster, - num_physical_clients: num_clients, - start_virtual_clients: 1, - virtual_clients_step: VIRTUAL_CLIENTS_STEP, - num_runs: 1, - calibrate_message_sizes: None, - }, - ) + (builder, program) } #[tokio::main] async fn main() { let args = Args::parse(); - for &write_ratio in WRITE_RATIOS { - benchmark_protocol( - BenchmarkArgs { - gcp: args.gcp.clone(), - aws: args.aws, - }, - || run_benchmark(write_ratio, args.optimize), - ) - .await; - } + let config = BenchmarkConfig { + name: "CAS".to_string(), + kind: if args.optimize { + Optimization::BottleneckElimination + } else { + Optimization::None + }, + num_physical_clients: NUM_PHYSICAL_CLIENTS, + start_virtual_clients: 1, + virtual_clients_step: 10, + num_runs: 1, + calibrate_message_sizes: None, + }; + // (params, workload name) — the name discriminates output directories on disk. + let workloads: Vec<(usize, String)> = WRITE_RATIOS + .iter() + .map(|&ratio| (ratio, format!("write{ratio}"))) + .collect(); + + benchmark_protocol( + BenchmarkArgs { + gcp: args.gcp.clone(), + aws: args.aws, + }, + config, + &workloads, + |&write_ratio| compile(write_ratio), + ) + .await; } diff --git a/hydro_optimize_examples/examples/network_calibrator.rs b/hydro_optimize_examples/examples/network_calibrator.rs index 4321b25..2e14867 100644 --- a/hydro_optimize_examples/examples/network_calibrator.rs +++ b/hydro_optimize_examples/examples/network_calibrator.rs @@ -6,8 +6,8 @@ use hydro_lang::{ prelude::{FlowBuilder, TCP}, }; use hydro_optimize::deploy_and_analyze::{ - BenchmarkArgs, BenchmarkConfig, Optimizations, ReusableClusters, ReusableProcesses, - benchmark_protocol, + BenchmarkArgs, BenchmarkConfig, CompiledProgram, Optimization, ReusableClusters, + ReusableProcesses, benchmark_protocol, }; use hydro_optimize_examples::network_calibrator::{Client, Server}; use stageleft::q; @@ -34,12 +34,24 @@ struct Args { async fn main() { let args = Args::parse(); + let config = BenchmarkConfig { + name: "Network".to_string(), + kind: Optimization::None, + num_physical_clients: 1, + start_virtual_clients: 1, + virtual_clients_step: 1, + num_runs: 1, + calibrate_message_sizes: Some(CALIBRATION_SIZES.to_vec()), + }; + benchmark_protocol( BenchmarkArgs { gcp: args.gcp.clone(), aws: args.aws, }, - move || { + config, + &[((), "default".to_string())], + move |_: &()| { let mut builder = FlowBuilder::new(); let server = builder.process::(); let clients = builder.cluster::(); @@ -60,25 +72,10 @@ async fn main() { let clusters = ReusableClusters::default().with_cluster(clients, 1); let processes = ReusableProcesses::default().with_process(server); - let optimizations = Optimizations::default() - .excluding(client_id); + let program = + CompiledProgram::new(clusters, processes, location_id_to_cluster).excluding(client_id); - ( - builder, - BenchmarkConfig { - name: "Network".to_string(), - workload: "default".to_string(), - clusters, - processes, - optimizations, - location_id_to_cluster, - num_physical_clients: 1, - start_virtual_clients: 1, - virtual_clients_step: 1, - num_runs: 1, - calibrate_message_sizes: Some(CALIBRATION_SIZES.to_vec()), - }, - ) + (builder, program) }, ) .await; From 43065332da398687b442f294ce9c20af57de3c27 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 30 Jun 2026 19:31:39 +0000 Subject: [PATCH 23/29] CAS fixes: Reads were labeled as blocked if there were no previous writes. Separate networking so decoupling is doesn't need to ping-pong messages. --- .../src/compare_and_swap/cas_distributed.rs | 358 ++++++++++++------ 1 file changed, 237 insertions(+), 121 deletions(-) diff --git a/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs b/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs index 43a343e..48c9abe 100644 --- a/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs +++ b/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs @@ -3,7 +3,7 @@ use hydro_lang::{ live_collections::{ boundedness::Boundedness, sliced::sliced, - stream::{AtLeastOnce, ExactlyOnce, NoOrder, Ordering, TotalOrder}, + stream::{ExactlyOnce, NoOrder, Ordering, TotalOrder}, }, location::{Location, MemberId, cluster::CLUSTER_SELF_ID}, nondet::nondet, @@ -107,6 +107,51 @@ impl<'a, 'b> DistributedCAS<'a, 'b> { } } + #[expect(clippy::type_complexity, reason = "Stream type")] + fn send_response( + &self, + response: KeyedStream< + MemberId, + ( + (Request, Ballot), + Option<(CASState, Ballot)>, + ), + Cluster<'a, Replica>, + Unbounded, + NoOrder, + >, + ) -> KeyedStream< + UniqueRequestId, + Response, + Cluster<'a, Replica>, + Unbounded, + NoOrder, + > + where + State: Serialize + for<'de> Deserialize<'de> + Clone + Debug + Eq + 'a, + Sender: Serialize + for<'de> Deserialize<'de> + Clone + Debug + 'a, + { + response + .entries() + .map(q!(|(sender, ((request, max_ballot), state_and_ballot))| { + ( + sender, + Response { + request, + max_ballot, + state_and_ballot, + }, + ) + })) + .demux(self.cluster, TCP.fail_stop().bincode()) + .values() + .map(q!(|response| { + // println!("Received response: {:?}", response); + (response.request.id, response) + })) + .into_keyed() + } + /// Returns the state associated with the respondent with the highest ballot, and whether all agree /// Returns only 1 response per request ID, on the tick the quorum is first reached. #[expect(clippy::type_complexity, reason = "Stream type")] @@ -140,9 +185,17 @@ impl<'a, 'b> DistributedCAS<'a, 'b> { .clone() .filter_key_not_in(no_quorum.into_keyed().keys()); let result = just_reached_quorum - .map(q!(|response| (response, true))) + .map(q!(|response| { + // if response.request.ballot.is_some() { + // println!("[CAS-TRACE 4] quorum reached id={:?} is_write={} not_preempted={}", + // response.request.id, response.request.state.is_some(), + // response.request.ballot.as_ref() == Some(&response.max_ballot)); + // } + (response, true) + })) .reduce(q!(|(prev_response, all_agree), (response, _)| { if prev_response.state_and_ballot.as_ref().map(|(state, _)| state) != response.state_and_ballot.as_ref().map(|(state, _)| state) { + println!("Warning: Responses disagree"); *all_agree = false; } // Return the state with the largest accepted ballot @@ -213,13 +266,15 @@ where let client_reads = self.send_to_cas(reads); let client_subscribes = self.send_to_cas(subscribe); - // These are ballots that we don't need to immediately take into account (is ok to delay) - let (incoming_response_ballots_complete, incoming_response_ballots) = self - .cluster - .forward_ref::, Unbounded, NoOrder, AtLeastOnce>>(); - let max_response_ballot = incoming_response_ballots.max(); - - let (incoming_requests_complete, incoming_requests) = + let (incoming_writes_complete, incoming_writes) = self.cluster.forward_ref::, + Request, + Cluster<'a, Replica>, + Unbounded, + NoOrder, + ExactlyOnce, + >>(); + let (incoming_elections_complete, incoming_elections) = self.cluster.forward_ref::, Request, @@ -228,16 +283,41 @@ where NoOrder, ExactlyOnce, >>(); - - // Quorum responses. bool = whether all agree on the highest ballot value - let (incoming_responses_complete, incoming_responses) = self.cluster.forward_ref::, bool), + let (incoming_reads_complete, incoming_reads) = self.cluster.forward_ref::, + Request, Cluster<'a, Replica>, Unbounded, NoOrder, ExactlyOnce, >>(); + // Quorum responses. bool = whether all agree on the highest ballot value + let (incoming_write_responses_complete, incoming_write_responses) = + self.cluster.forward_ref::, bool), + Cluster<'a, Replica>, + Unbounded, + NoOrder, + ExactlyOnce, + >>(); + let (incoming_election_responses_complete, incoming_election_responses) = + self.cluster.forward_ref::, bool), + Cluster<'a, Replica>, + Unbounded, + NoOrder, + ExactlyOnce, + >>(); + let (incoming_blocked_read_responses_complete, incoming_blocked_read_responses) = + self.cluster.forward_ref::, bool), + Cluster<'a, Replica>, + Unbounded, + NoOrder, + ExactlyOnce, + >>(); + // ------------------------------------------------------------- // Election and state // @@ -254,11 +334,22 @@ where nondet!(/** Randomly decide when to retry election */), ); let nondet_ballot = nondet!(/** The ballot used depends on the time of message arrival */); - let (write_requests, election_requests, responses, unblocked_reads, invalid_writes) = sliced! { + let ( + write_requests, + election_requests, + write_responses, + election_responses, + read_responses, + unblocked_reads, + invalid_writes, + ) = sliced! { let client_writes = use(client_writes.entries(), nondet_ballot); - let incoming_requests = use(incoming_requests, nondet_ballot); - let max_response_ballot = use(max_response_ballot, nondet_ballot); - let incoming_responses = use(incoming_responses, nondet_ballot); + let incoming_reads = use(incoming_reads, nondet_ballot); + let incoming_elections = use(incoming_elections, nondet_ballot); + let incoming_writes = use(incoming_writes, nondet_ballot); + let incoming_write_responses = use(incoming_write_responses, nondet_ballot); + let incoming_election_responses = use(incoming_election_responses, nondet_ballot); + let incoming_blocked_read_responses = use(incoming_blocked_read_responses, nondet_ballot); let election_timeout = use(election_timeout, nondet_ballot); let mut retry_election_count = use::state::>(|l| l.singleton(q!(0))); let mut write_ballot = use::state::>(|l| l.singleton(q!(Ballot { @@ -275,9 +366,18 @@ where // Reads blocking on reconciliation let mut blocked_reads = use::state_null::, _, Bounded, NoOrder>>(); - let request_ballots = incoming_requests.clone().values().filter_map(q!(|request| request.ballot)); + let incoming_requests = incoming_reads.clone() + .chain(incoming_elections.clone()) + .chain(incoming_writes.clone()); + let request_ballots = incoming_requests.values().filter_map(q!(|request| { + request.ballot + })); + let incoming_responses = incoming_write_responses.clone() + .chain(incoming_election_responses.clone()) + .chain(incoming_blocked_read_responses.clone()); + let response_ballots = incoming_responses.map(q!(|(response, _all_agree)| response.max_ballot)); let max_ballot_no_default = request_ballots - .chain(max_response_ballot.into_stream()) + .chain(response_ballots) .max() .into_singleton(); let max_ballot = max_ballot_no_default @@ -288,41 +388,46 @@ where }))); // Incoming responses will contain N blocked_reads and either 1 election or write response, if any. - let (read_responses, election_or_write_responses) = incoming_responses - .clone() - .partition(q!(|(response, _all_agree)| response.request.ballot.is_none())); - let (election_responses, write_responses) = election_or_write_responses - .partition(q!(|(response, _all_agree)| response.request.state.is_none())); - let (write_successes, write_failures) = write_responses - .partition(q!(|(response, _all_agree)| response.request.ballot.clone().expect("write response missing ballot") == response.max_ballot)); + let (write_successes, write_failures) = incoming_write_responses + .partition(q!(|(response, _all_agree)| + response.request.ballot.clone().expect("write response missing ballot") == response.max_ballot + )); // 1. Reads responses // Unblock reads if we observe a committed state that is higher than the ballot they observed - let all_blocked_reads = read_responses + let all_blocked_reads = incoming_blocked_read_responses .map(q!(|(response, _all_agree)| (response.request.id, response))) .into_keyed() .chain(blocked_reads.clone()); let unblocked_reads = all_blocked_reads .clone() .cross_singleton(committed_state.clone()) - .filter_map(q!(|(response, (ballot, state))| - (ballot >= response.max_ballot) + .filter_map(q!(|(response, (ballot, state))| { + let unblocked = ballot >= response.max_ballot; + unblocked .then(|| (response.request.client_id.expect("read response missing client_id"), (response.request.id, state.clone()))) - )); + })); let new_blocked_reads = all_blocked_reads.filter_key_not_in(unblocked_reads.clone().keys()); // 2. Election responses - let (won_election, lost_election) = election_responses + let (won_election, lost_election) = incoming_election_responses .clone() - .partition(q!(move |(response, _all_agree)| response.max_ballot.node == CLUSTER_SELF_ID)); + .partition(q!(move |(response, _all_agree)| { + response.max_ballot.node == CLUSTER_SELF_ID + })); let (won_election_write_complete, won_election_write_uncommitted) = won_election .partition(q!(|(_response, all_agree)| *all_agree)); // Propose reconciling write if some replicas disagree let proposed_reconciling_write = won_election_write_uncommitted - .map(q!(|(response, _all_agree)| (response.request.id, (None, response.state_and_ballot.map(|(state, _ballot)| state).expect("uncommitted election response missing state"))))); + .map(q!(|(response, _all_agree)| { + (response.request.id, (None, response.state_and_ballot.map(|(state, _ballot)| state).expect("uncommitted election response missing state"))) + })); let new_committed_state = won_election_write_complete .clone() - .map(q!(|(response, _all_agree)| (response.max_ballot, response.state_and_ballot.map(|(state, _ballot)| state)))) + .map(q!(|(response, _all_agree)| { + // println!("[CAS-TRACE 5a] won election (all agree), ballot={:?}", response.max_ballot); + (response.max_ballot, response.state_and_ballot.map(|(state, _ballot)| state)) + })) .chain(committed_state.into_stream()) .max(); // Propose a valid client write if the election is succesful and all agree @@ -334,6 +439,7 @@ where // Allow write if version is prev+1 or the writer is the same and the version hasn't changed if response.state_and_ballot.is_none_or(|(curr_state, _curr_ballot)| curr_state.version + 1 == state.version || (curr_state.writer == state.writer && curr_state.version == state.version)) { + // println!("[CAS-TRACE 6] proposing client write id={:?}", request_id); Some((request_id, (Some(client_id), state))) } else { @@ -349,7 +455,7 @@ where .entries() .cross_singleton(new_committed_state.clone()) .filter_map(q!(|((request_id, (client_id, state)), (_ballot, committed_state))| - committed_state.is_none_or(|committed| committed.version > state.version).then_some((client_id, (request_id, false))))); + committed_state.is_some_and(|committed| committed.version > state.version).then_some((client_id, (request_id, false))))); let invalid_write_ids = invalid_writes .clone() .map(q!(|(_client_id, (request_id, _successful))| request_id)); @@ -366,7 +472,10 @@ where let new_write_queue = write_queue .clone() .filter_key_not_in(successful_write.chain(invalid_write_ids)) - .chain(client_writes.clone().map(q!(|(client_id, (request_id, state))| (request_id, (client_id, state)))).into_keyed()); + .chain(client_writes.clone().map(q!(|(client_id, (request_id, state))| { + // println!("[CAS-TRACE 1] received client write id={:?}", request_id); + (request_id, (client_id, state)) + })).into_keyed()); let had_no_writes_or_uncommitted_reads = write_queue .entries() .is_empty() @@ -387,10 +496,14 @@ where .into_singleton() .map(q!(|count| count.is_some_and(|c| c == 1))) .and(retry_election_count.clone().map(q!(|count| count == 0))); + let ready_for_election = ready_to_retry_election.or(ready_for_consecutive_election.or(ready_for_new_election)); // If we lost the election, or if our writes were preempted, retry after election_timeout retry_election_count = lost_election .chain(write_failures) - .map(q!(|_| 2)) // Only allow election after 2 more timeouts (2 in case the timeout immediately goes off) + .map(q!(|_| { + println!("Retrying after timeout"); + 2 + })) // Only allow election after 2 more timeouts (2 in case the timeout immediately goes off) .assume_ordering::(nondet!(/** Only 1 outgoing election at a time*/)) .first() .unwrap_or(retry_election_count) @@ -399,17 +512,17 @@ where let new_ballot = max_ballot .clone() .map(q!(move |ballot| Ballot { num: ballot.num + 1, node: CLUSTER_SELF_ID.clone() })) - .filter_if(ready_to_retry_election.clone().or(ready_for_new_election.clone())); + .filter_if(ready_for_election.clone()); write_ballot = new_ballot.unwrap_or(write_ballot); // Determine local state based on incoming write requests - let winning_write = incoming_requests + let winning_write = incoming_writes .clone() .values() .reduce(q!(|max_req, request| { - // Find the max ballot request with state + // Find the max ballot request if let Some(old_ballot) = &max_req.ballot { - if let Some(ref new_ballot) = request.ballot && request.state.is_some() && *old_ballot <= *new_ballot { + if let Some(ref new_ballot) = request.ballot && *old_ballot <= *new_ballot { *max_req = request; } } @@ -431,18 +544,25 @@ where .cross_singleton(write_ballot.clone()); let election_requests = write_ballot .clone() - .filter_if(ready_to_retry_election.or(ready_for_new_election).or(ready_for_consecutive_election)) + .filter_if(ready_for_election) .into_stream(); - let responses = incoming_requests + let next_state_singleton = next_state.clone().into_singleton(); + let write_responses = incoming_writes + .cross_singleton(max_ballot.clone()) + .cross_singleton(next_state_singleton.clone()); + let election_responses = incoming_elections + .cross_singleton(max_ballot.clone()) + .cross_singleton(next_state_singleton.clone()); + let read_responses = incoming_reads .cross_singleton(max_ballot) - .cross_singleton(next_state.clone().into_singleton()); + .cross_singleton(next_state_singleton); state = next_state; blocked_reads = new_blocked_reads; committed_state = new_committed_state; write_queue = new_write_queue; - (write_requests, election_requests, responses, unblocked_reads.values(), invalid_writes) + (write_requests, election_requests, write_responses, election_responses, read_responses, unblocked_reads.values(), invalid_writes) }; // ------------------------------------------------------------- @@ -461,107 +581,110 @@ where TCP.fail_stop().bincode(), nondet!(/** static membership */), ); + incoming_reads_complete.complete(sent_reads); let sent_elections = election_requests - .map(q!(|ballot| Request { - id: UniqueRequestId::generate(), - client_id: None, - ballot: Some(ballot), - state: None, + .map(q!(|ballot| { + Request { + id: UniqueRequestId::generate(), + client_id: None, + ballot: Some(ballot), + state: None, + } })) .broadcast( self.cluster, TCP.fail_stop().bincode(), nondet!(/** static membership */), ); + incoming_elections_complete.complete(sent_elections); let sent_writes = write_requests - .map(q!(|((request_id, (client_id, write)), ballot)| Request { - id: request_id, - client_id, - ballot: Some(ballot), - state: Some(write), + .map(q!(|((request_id, (client_id, write)), ballot)| { + // println!("[CAS-TRACE 3] broadcasting WRITE id={:?} ballot={:?} client={:?}", request_id, ballot, client_id); + Request { + id: request_id, + client_id, + ballot: Some(ballot), + state: Some(write), + } })) .broadcast( self.cluster, TCP.fail_stop().bincode(), nondet!(/** static membership */), ); - let sent_requests = sent_reads - .merge_unordered(sent_elections) - .merge_unordered(sent_writes); - incoming_requests_complete.complete(sent_requests); + incoming_writes_complete.complete(sent_writes); // ------------------------------------------------------------- // Reply to requests with responses. // ------------------------------------------------------------- - let sent_responses = responses - .entries() - .map(q!(|(sender, ((request, max_ballot), state_and_ballot))| { - ( - sender, - Response { - request, - max_ballot, - state_and_ballot, - }, - ) - })) - .demux(self.cluster, TCP.fail_stop().bincode()) - .values() - .map(q!(|response| (response.request.id, response))) - .into_keyed(); - + let sent_write_response = self.send_response(write_responses); + let sent_election_response = self.send_response(election_responses); + let sent_read_response = self.send_response(read_responses); // ------------------------------------------------------------- // Process responses. // ------------------------------------------------------------- - let responses_agree = self.check_all_agree(sent_responses); - incoming_responses_complete.complete(responses_agree.clone()); + let write_responses_agree = self.check_all_agree(sent_write_response); + incoming_write_responses_complete.complete(write_responses_agree.clone()); + let election_responses_agree = self.check_all_agree(sent_election_response); + incoming_election_responses_complete.complete(election_responses_agree.clone()); + let read_responses_agree = self.check_all_agree(sent_read_response); + // Only block reads where some disagree + incoming_blocked_read_responses_complete.complete( + read_responses_agree + .clone() + .filter(q!(|(_response, all_agree)| !all_agree)), + ); // Reads are responses without a ballot - let read_result = responses_agree + let read_result_to_clients = read_responses_agree .clone() - .filter_map(q!(|(response, all_agree)| (response - .request - .ballot - .is_none() // If this is a read - && response.request.state.is_none_or(|_state| all_agree)) // And either all agree or there's no state - .then(|| ( - response - .request - .client_id - .expect("read result missing client_id"), - ( - response.request.id, - response.state_and_ballot.map(|(state, _ballot)| state) - ) - )))) - .merge_unordered(unblocked_reads) + .filter_map(q!(|(response, all_agree)| { + // Either all agree or there's no state + (response.request.state.is_none_or(|_state| all_agree)).then(|| { + ( + response + .request + .client_id + .expect("read result missing client_id"), + ( + response.request.id, + response.state_and_ballot.map(|(state, _ballot)| state), + ), + ) + }) + })) + .merge_unordered(unblocked_reads); + let read_result = read_result_to_clients .demux(sender, TCP.fail_stop().bincode()) .values() .into_keyed(); - let write_success = responses_agree - .clone() - .filter_map(q!(|(response, _all_agree)| (response + let write_success = + write_responses_agree + .clone() + .filter_map(q!(|(response, _all_agree)| (response .request .ballot .is_some_and(|ballot| ballot == response.max_ballot) // If we weren't preempted - && response.request.state.is_some() // And this is a write (not election) && response.request.client_id.is_some()) // And this is not a reconciling write - .then(|| ( - response - .request - .client_id - .expect("write success missing client_id"), - response.request.id, - response - .state_and_ballot - .map(|(state, _ballot)| state) - .expect("write success missing state") - )))); + .then(|| { + // println!("[CAS-TRACE 7] WRITE SUCCESS id={:?}", response.request.id); + ( + response + .request + .client_id + .expect("write success missing client_id"), + response.request.id, + response + .state_and_ballot + .map(|(state, _ballot)| state) + .expect("write success missing state"), + ) + }))); let write_processed = write_success .clone() - .map(q!(|(client_id, request_id, _state)| ( - client_id, - (request_id, true) - ))) + .map(q!(|(client_id, request_id, _state)| { + // println!("[CAS-TRACE 8] write ACK -> client id={:?}", request_id); + (client_id, (request_id, true)) + })) .merge_unordered(invalid_writes) .demux(sender, TCP.fail_stop().bincode()) .values() @@ -590,13 +713,6 @@ where .demux(sender, TCP.fail_stop().bincode()) .values(); - // ------------------------------------------------------------- - // Gather all ballots received as a response. - // ------------------------------------------------------------- - let received_ballots = - responses_agree.map(q!(|(response, _all_agree)| response.max_ballot)); - incoming_response_ballots_complete.complete(received_ballots); - CASOutput { read_result, write_processed, From ccd961d8618e64137132df18e702bde7a7a9a385 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 30 Jun 2026 22:03:33 +0000 Subject: [PATCH 24/29] Fix send from partitions own ID bug --- hydro_optimize/src/deploy_and_analyze.rs | 2 -- hydro_optimize/src/rewriter.rs | 13 +++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hydro_optimize/src/deploy_and_analyze.rs b/hydro_optimize/src/deploy_and_analyze.rs index 6981196..935a13b 100644 --- a/hydro_optimize/src/deploy_and_analyze.rs +++ b/hydro_optimize/src/deploy_and_analyze.rs @@ -837,8 +837,6 @@ async fn run_scaling_loop<'a>( } else { throughput_sum += run_throughput; successful_runs += 1; - // TODO: Remove after debugging CAS - // break 'outer; } run += 1; final_run_metadata = run_metadata; diff --git a/hydro_optimize/src/rewriter.rs b/hydro_optimize/src/rewriter.rs index cf30fda..aa2c4ec 100644 --- a/hydro_optimize/src/rewriter.rs +++ b/hydro_optimize/src/rewriter.rs @@ -86,7 +86,10 @@ fn map_before_network(node: &mut HydroNode, network_metadata: &NetworkMetadata) // If this is an existing network, then the index is over fields of the network after the receipt(sender, T), but before the send, the type is just T, so lop off the 1st layer of the field. let field: &[String] = if !network_metadata.new { // Partitioning on sender location CAN be supported, but it will involve introducing a Hash over CLUSTER_SELF_ID - assert_eq!(field[0], "1", "Unsupported: Partitioning on sender location."); + assert_eq!( + field[0], "1", + "Unsupported: Partitioning on sender location." + ); &field[1..] } else { field @@ -122,9 +125,15 @@ fn map_before_network(node: &mut HydroNode, network_metadata: &NetworkMetadata) ), proc_macro2::Span::call_site(), ); + let num_sender_partitions = network_metadata.sender_partitions; + let orig_raw_expr: syn::Expr = if num_sender_partitions > 0 { + syn::parse_quote!(#ident.get_raw_id() / #num_sender_partitions as u32) + } else { + syn::parse_quote!(#ident.get_raw_id()) + }; syn::parse_quote!( |struct_or_tuple: #element_type| { - let orig_raw = #ident.get_raw_id(); + let orig_raw = #orig_raw_expr; let dest = #dest_expr; (hydro_lang::location::MemberId::<()>::from_raw_id(dest), struct_or_tuple) } From 6609377dcc0e090f468d27e83818ec1595302c7f Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 30 Jun 2026 23:41:27 +0000 Subject: [PATCH 25/29] Prioritize partitioning over less specific fields for uniform distribution --- hydro_optimize/src/decouple_analysis.rs | 14 ++++++++++++-- hydro_optimize/src/partition_ilp_analysis.rs | 11 ++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/hydro_optimize/src/decouple_analysis.rs b/hydro_optimize/src/decouple_analysis.rs index 543faa0..814f601 100644 --- a/hydro_optimize/src/decouple_analysis.rs +++ b/hydro_optimize/src/decouple_analysis.rs @@ -9,7 +9,10 @@ use crate::deploy_and_analyze::MAX_BUDGET_PER_CLUSTER; use crate::parse_results::{NetworkCostTable, SarStats}; use crate::partition_ilp_analysis::{apply_budget_constraints, partition_ilp_analysis}; use crate::partition_syn_analysis::StructOrTupleIndex; -use crate::rewrites::{all_inputs, get_tick_id, is_serializable, is_syntactic_sugar, nodes_dependent_on_inputs, op_id_to_parents}; +use crate::rewrites::{ + all_inputs, get_tick_id, is_serializable, is_syntactic_sugar, nodes_dependent_on_inputs, + op_id_to_parents, +}; use good_lp::solvers::lp_solvers::{GurobiSolver, LpSolution, LpSolver}; use good_lp::{ Constraint, Expression, ProblemVariables, Solution, SolverModel, Variable, constraint, @@ -36,6 +39,7 @@ pub fn num_to_alpha(n: usize) -> String { // Penalty for decoupling regardless of cardinality (to prevent decoupling low cardinality operators) const DECOUPLING_PENALTY: f64 = 0.0001; const LEXICOGRAPHIC_EPSILON: f64 = 0.0001; // Tiebreaker weight to minimize non-bottleneck locations +const FIELD_SPECIFICITY_EPSILON: f64 = LEXICOGRAPHIC_EPSILON * 0.001; // Smaller tiebreaker to prefer broader partition fields /// Each operator is assigned either 0 or 1 /// 0 means that its output will go to the original node, 1 means that it will go to the decoupled node @@ -69,6 +73,8 @@ pub(crate) struct DecoupleILPMetadata { prev_op_parent_with_tick: HashMap, // tick_id: last op_id with that tick_id tee_inner_to_decoupled_vars: HashMap>, /* inner_id: (loc: (send var, recv var)) */ pub(crate) network_ids: HashMap, + // Add small penalties for partitioning on too-specific fields, since those are less likely to be uniformly distributed + pub(crate) field_specificity_penalty: Expression, } /// Per-location ILP expressions for each resource type. @@ -568,6 +574,7 @@ fn solve(decoupling_metadata: &RefCell) -> LpSolution { resource_usages, op_id_to_var, max_num_locations, + field_specificity_penalty, .. } = &mut *decoupling_metadata.borrow_mut(); @@ -602,7 +609,9 @@ fn solve(decoupling_metadata: &RefCell) -> LpSolution { } } - let objective = highest_saturation + LEXICOGRAPHIC_EPSILON * sum_saturations; + let objective = highest_saturation + + LEXICOGRAPHIC_EPSILON * sum_saturations + + FIELD_SPECIFICITY_EPSILON * field_specificity_penalty.clone(); println!( " Solving ILP: {} vars, {} constraints", @@ -734,6 +743,7 @@ pub(crate) fn decouple_analysis( prev_op_parent_with_tick: HashMap::new(), tee_inner_to_decoupled_vars: HashMap::new(), network_ids: HashMap::new(), + field_specificity_penalty: Expression::default(), }); let op_id_to_parents = op_id_to_parents(ir, Some(bottleneck), cycle_source_to_sink_parent); let inputs = all_inputs(ir, &decoupling_metadata.borrow().bottleneck); diff --git a/hydro_optimize/src/partition_ilp_analysis.rs b/hydro_optimize/src/partition_ilp_analysis.rs index 8cfd83a..fc7a5cb 100644 --- a/hydro_optimize/src/partition_ilp_analysis.rs +++ b/hydro_optimize/src/partition_ilp_analysis.rs @@ -414,6 +414,10 @@ pub(crate) struct PartitionILPMetadata { pub(crate) can_partition: HashMap, // location: 1 iff location is partitionable ((all relevant partitionable OR no persist) AND no total-order) } +fn field_specificity_score(field: &StructOrTupleIndex) -> f64 { + (field.len() + 1) as f64 +} + /// Add the operator with `id` to the location_sum for each location fn add_op_to_location_sum( id: usize, @@ -494,14 +498,19 @@ fn field_vars_from_op( let mut field_to_var = HashMap::new(); let mut sum_expr = Expression::default(); for (idx, field_name) in field_names.iter().enumerate() { + let mut decoupling_metadata = decoupling_metadata.borrow_mut(); let var = decoupling_metadata - .borrow_mut() .variables .add(variable().binary().name(format!( "fieldop{}f{}", num_to_alpha(op_id), num_to_alpha(idx) ))); + // Add penality for partitioning on too-specific fields + let penalty = std::mem::take(&mut decoupling_metadata.field_specificity_penalty); + decoupling_metadata.field_specificity_penalty = + penalty + Expression::from(var) * field_specificity_score(field_name); + field_to_var.insert(field_name.clone(), var); sum_expr += var; } From 4b692dabedd2bc1aecaa25d0228eae84e32483e3 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Tue, 30 Jun 2026 23:41:48 +0000 Subject: [PATCH 26/29] Implement hash for partitioning --- .../src/compare_and_swap/cas_distributed.rs | 6 +++--- hydro_optimize_examples/src/compare_and_swap/cas_like.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs b/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs index 48c9abe..71cb831 100644 --- a/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs +++ b/hydro_optimize_examples/src/compare_and_swap/cas_distributed.rs @@ -26,7 +26,7 @@ pub struct DistributedCAS<'a, 'b> { pub retry_timeout: u64, } -#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Hash)] pub struct Ballot { pub num: u64, pub node: MemberId, @@ -49,7 +49,7 @@ impl Ord for Ballot { /// No `client_id`: Election request /// No `ballot`: Read request /// No `state`: Read and election request -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] pub struct Request { pub id: UniqueRequestId, pub client_id: Option>, @@ -75,7 +75,7 @@ impl Ord for Request { } /// - `max_ballot`: The max ballot seen by the respondent, which the leader can use to determine if it has been preempted. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Hash)] pub struct Response { pub request: Request, pub max_ballot: Ballot, diff --git a/hydro_optimize_examples/src/compare_and_swap/cas_like.rs b/hydro_optimize_examples/src/compare_and_swap/cas_like.rs index 43eeaef..6b3e107 100644 --- a/hydro_optimize_examples/src/compare_and_swap/cas_like.rs +++ b/hydro_optimize_examples/src/compare_and_swap/cas_like.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; /// Version-guarded write request. Accepted only if /// `version = current_version + 1` in the register. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] pub struct CASState { pub version: u64, pub state: State, From 3780f2e8a51729bd6ad184e810b144607e799fd4 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Wed, 1 Jul 2026 19:19:53 +0000 Subject: [PATCH 27/29] Support partitioning on sender ID --- hydro_optimize/src/rewriter.rs | 73 ++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/hydro_optimize/src/rewriter.rs b/hydro_optimize/src/rewriter.rs index aa2c4ec..11eccd9 100644 --- a/hydro_optimize/src/rewriter.rs +++ b/hydro_optimize/src/rewriter.rs @@ -68,6 +68,30 @@ struct NetworkMetadata { new: bool, } +fn sender_logical_id(network_metadata: &NetworkMetadata) -> syn::Expr { + let ident = syn::Ident::new( + &format!( + "__hydro_lang_cluster_self_id_{}", + network_metadata.sender_location.key() + ), + proc_macro2::Span::call_site(), + ); + let num_sender_partitions = network_metadata.sender_partitions; + if num_sender_partitions > 0 { + syn::parse_quote!(#ident.get_raw_id() / #num_sender_partitions as u32) + } else { + syn::parse_quote!(#ident.get_raw_id()) + } +} + +fn hash_expr(value: syn::Expr) -> syn::Expr { + syn::parse_quote!({ + let mut s = ::std::hash::DefaultHasher::new(); + ::std::hash::Hash::hash(&#value, &mut s); + ::std::hash::Hasher::finish(&s) as u32 + }) +} + /// Creates the Map before Network to route it to the correct partition. /// A map is appended to the node passed in. /// If a network already exists, pass in its input. @@ -83,23 +107,26 @@ fn map_before_network(node: &mut HydroNode, network_metadata: &NetworkMetadata) if let Some(field) = &network_metadata.partition_field { // If partitioning and there is a field to hash on, use it let struct_or_tuple: syn::Expr = syn::parse_quote! { struct_or_tuple }; - // If this is an existing network, then the index is over fields of the network after the receipt(sender, T), but before the send, the type is just T, so lop off the 1st layer of the field. - let field: &[String] = if !network_metadata.new { - // Partitioning on sender location CAN be supported, but it will involve introducing a Hash over CLUSTER_SELF_ID - assert_eq!( - field[0], "1", - "Unsupported: Partitioning on sender location." - ); - &field[1..] + if !network_metadata.new && field.len() == 1 && field[0] == "0" { + // Partitioning asks to hash on the Sender's ID + hash_expr(sender_logical_id(network_metadata)) } else { - field - }; - let struct_or_tuple_with_fields = StructOrTuple::to_syn_expr(struct_or_tuple, field); - syn::parse_quote!({ - let mut s = ::std::hash::DefaultHasher::new(); - ::std::hash::Hash::hash(&#struct_or_tuple_with_fields, &mut s); - ::std::hash::Hasher::finish(&s) as u32 - }) + // If this is an existing network, then the index is over fields of the network + // after the receipt(sender, T), but before the send, the type is just T, so lop + // off the first layer of the field. + let field: &[String] = if !network_metadata.new { + assert!( + !field.is_empty() && field[0] == "1", + "Unsupported: Existing network partition fields must refer to either sender id or payload." + ); + &field[1..] + } else { + field + }; + let struct_or_tuple_with_fields = + StructOrTuple::to_syn_expr(struct_or_tuple, field); + hash_expr(struct_or_tuple_with_fields) + } } else { // If partitioning and there is no field to hash on, we must be random partitioning syn::parse_quote!(::rand::random::()) @@ -118,19 +145,7 @@ fn map_before_network(node: &mut HydroNode, network_metadata: &NetworkMetadata) let f: syn::Expr = if network_metadata.new { // New network type is just T. We need (receiver MemberId, T) - let ident = syn::Ident::new( - &format!( - "__hydro_lang_cluster_self_id_{}", - network_metadata.sender_location.key() - ), - proc_macro2::Span::call_site(), - ); - let num_sender_partitions = network_metadata.sender_partitions; - let orig_raw_expr: syn::Expr = if num_sender_partitions > 0 { - syn::parse_quote!(#ident.get_raw_id() / #num_sender_partitions as u32) - } else { - syn::parse_quote!(#ident.get_raw_id()) - }; + let orig_raw_expr = sender_logical_id(network_metadata); syn::parse_quote!( |struct_or_tuple: #element_type| { let orig_raw = #orig_raw_expr; From ba06f8e6ef5e1681898e23ceef33a6d037164615 Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Wed, 1 Jul 2026 19:20:36 +0000 Subject: [PATCH 28/29] Plotting script --- scripts/plot_throughput_latency.py | 231 +++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100755 scripts/plot_throughput_latency.py diff --git a/scripts/plot_throughput_latency.py b/scripts/plot_throughput_latency.py new file mode 100755 index 0000000..aec6186 --- /dev/null +++ b/scripts/plot_throughput_latency.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Plot throughput-latency curves across optimization configurations. + +Examples: + scripts/plot_throughput_latency.py Paxos + scripts/plot_throughput_latency.py CAS write100 + scripts/plot_throughput_latency.py CAS --workload write0 --metric p99 + +If deployment produced repeated runs for the same client count, the plot uses +the latest run index because earlier runs may be stabilization attempts. +""" + +import argparse +import csv +import glob +import os +import re +from collections import defaultdict + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +plt.rcParams.update( + { + "font.size": 15, + "axes.labelsize": 15, + "xtick.labelsize": 13, + "ytick.labelsize": 13, + "legend.fontsize": 12, + "legend.title_fontsize": 12, + } +) + +RESULTS_DIR = os.path.join(os.path.dirname(__file__), "..", "benchmark_results") +START_S, END_S = 30, 59 # matches START_MEASUREMENT_SECOND / MEASUREMENT_SECOND + +FNAME_RE = re.compile(r"client_aggregator_(\d+)c_(\d+)vc_r(\d+)\.csv$") +OPT_RE = re.compile(r"opt(\d+)$") + + +def config_sort_key(config): + if config == "default": + return (0, 0) + match = OPT_RE.fullmatch(config) + if match: + return (1, int(match.group(1))) + return (2, config) + + +def parse_result_dir(protocol, name): + if not name.startswith(f"{protocol}_"): + return None + + parts = name.split("_") + if len(parts) < 3: + return None + + suffix = parts[-1] + if suffix != "none": + return None + + if parts[1] == "default": + return "default", "default" + + if OPT_RE.fullmatch(parts[1]): + if len(parts) < 4: + return None + return parts[1], "_".join(parts[2:-1]) + + return "default", "_".join(parts[1:-1]) + + +def find_config_dirs(protocol, workload): + dirs = [] + for path in glob.glob(os.path.join(RESULTS_DIR, f"{protocol}_*")): + if not os.path.isdir(path): + continue + parsed = parse_result_dir(protocol, os.path.basename(path)) + if parsed is None: + continue + config, dir_workload = parsed + if dir_workload == workload: + dirs.append((config, path)) + return sorted(dirs, key=lambda item: config_sort_key(item[0])) + + +def window_avg(path): + """Average throughput and latency percentiles over the steady-state window.""" + with open(path) as f: + rows = list(csv.DictReader(f)) + sel = [r for r in rows if START_S <= int(r["time_s"]) <= END_S] + if not sel: + return None + n = len(sel) + return { + "throughput": sum(float(r["throughput_rps"]) for r in sel) / n, + "p50": sum(float(r["latency_p50_ms"]) for r in sel) / n, + "p99": sum(float(r["latency_p99_ms"]) for r in sel) / n, + "p999": sum(float(r["latency_p999_ms"]) for r in sel) / n, + } + + +def collect_config(path): + """Return one point per virtual-client count, using the latest repeated run.""" + by_vc = defaultdict(list) + for csv_path in glob.glob(os.path.join(path, "client_aggregator_*.csv")): + match = FNAME_RE.search(os.path.basename(csv_path)) + if not match: + continue + physical_clients = int(match.group(1)) + virtual_clients = int(match.group(2)) + run_idx = int(match.group(3)) + agg = window_avg(csv_path) + if agg is None: + continue + agg["vc"] = virtual_clients + agg["clients"] = physical_clients * virtual_clients + agg["run"] = run_idx + by_vc[virtual_clients].append(agg) + + points = [] + for virtual_clients, runs in by_vc.items(): + latest = max(runs, key=lambda p: p["run"]) + points.append(latest) + + return sorted(points, key=lambda p: p["vc"]) + + +def output_path(protocol, workload, metric, explicit_output): + if explicit_output: + return explicit_output + basename = f"{protocol.lower()}_{workload}_throughput_latency_{metric}.png" + return os.path.join(os.path.dirname(__file__), basename) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Plot throughput-latency curves for default/opt benchmark configs." + ) + parser.add_argument("protocol", help="Protocol name, e.g. Paxos or CAS.") + parser.add_argument( + "workload_pos", + nargs="?", + help="Workload name, e.g. default, write0, or write100.", + ) + parser.add_argument( + "--workload", + default=None, + help="Workload name. Overrides the optional positional workload.", + ) + parser.add_argument( + "--metric", + choices=["p50", "p99", "p999"], + default="p50", + help="Latency percentile to plot.", + ) + parser.add_argument( + "--output", + default=None, + help="Output PNG path. Defaults to scripts/__throughput_latency_.png.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + protocol = args.protocol + workload = args.workload or args.workload_pos or "default" + metric = args.metric + + config_dirs = find_config_dirs(protocol, workload) + if not config_dirs: + raise SystemExit( + f"No benchmark result directories found for protocol={protocol!r}, workload={workload!r}" + ) + + data = [(config, collect_config(path)) for config, path in config_dirs] + data = [(config, points) for config, points in data if points] + if not data: + raise SystemExit( + f"No client_aggregator CSV data found for protocol={protocol!r}, workload={workload!r}" + ) + + fig, ax = plt.subplots(figsize=(8, 4.5)) + cmap = plt.get_cmap("tab10") + + for idx, (config, points) in enumerate(data): + xs = [p["throughput"] / 1000 for p in points] + ys = [p[metric] for p in points] + ax.plot( + xs, + ys, + "-o", + color=cmap(idx % 10), + label=config, + markersize=4, + linewidth=1.5, + ) + last = points[-1] + ax.annotate( + f"{int(last['clients'])}c", + (last["throughput"], last[metric]), + fontsize=7, + xytext=(4, 4), + textcoords="offset points", + ) + + ax.set_xlabel("Throughput (thousands of requests/s)") + ax.set_ylabel(f"{metric} latency (ms)") + ax.grid(True, alpha=0.3) + ax.legend(title="Configuration") + fig.tight_layout() + + out = output_path(protocol, workload, metric, args.output) + fig.savefig(out, dpi=150) + print(f"Wrote {out}") + + for config, points in data: + print(f"\n=== {config} ===") + print(f"{'clients':>8} {'run':>4} {'thr_rps':>10} {'p50_ms':>8} {'p99_ms':>8}") + for p in points: + print( + f"{int(p['clients']):>8} {p['run']:>4} {p['throughput']:>10.0f} " + f"{p['p50']:>8.3f} {p['p99']:>8.3f}" + ) + + +if __name__ == "__main__": + main() From 41be723a210c0b05679f415a5521fe07f2a1afdb Mon Sep 17 00:00:00 2001 From: David Chu McElroy Date: Wed, 1 Jul 2026 19:38:41 +0000 Subject: [PATCH 29/29] cargo fmt + clippy --- hydro_optimize/src/decouple_analysis.rs | 1 + hydro_optimize/src/parse_results.rs | 15 ++++++++++++--- hydro_optimize/src/partition_syn_analysis.rs | 10 ++-------- hydro_optimize/src/rewrites.rs | 5 +++-- hydro_optimize_examples/examples/cas.rs | 4 ++-- .../examples/network_calibrator.rs | 4 ++-- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/hydro_optimize/src/decouple_analysis.rs b/hydro_optimize/src/decouple_analysis.rs index 814f601..4aa2daa 100644 --- a/hydro_optimize/src/decouple_analysis.rs +++ b/hydro_optimize/src/decouple_analysis.rs @@ -1006,6 +1006,7 @@ pub fn find_optimal_budget( /// If this is under 100%, then our calibration assumes network costs too little. /// /// The return type is `Vec` just to match the signature of `find_optimal_budget` +#[allow(dead_code)] pub(crate) fn project_total_cpu( ir: &mut [HydroRoot], bottleneck: &LocationId, diff --git a/hydro_optimize/src/parse_results.rs b/hydro_optimize/src/parse_results.rs index b4f6f43..2554e94 100644 --- a/hydro_optimize/src/parse_results.rs +++ b/hydro_optimize/src/parse_results.rs @@ -23,6 +23,7 @@ use crate::deploy_and_analyze::{ /// 1. Finds the run suffix (e.g. "10c_500vc_r0") with highest avg throughput. /// 2. For that run, reads all cluster CSVs and returns the cluster name whose SAR stats /// are closest to saturation (100% CPU, 100% memory, or at/above max network/IO). +/// /// For one run directory, returns each cluster's saturation (max resource utilization) at the /// highest-throughput run suffix. Returns an empty map if the directory has no CSVs. fn cluster_saturations_from_run( @@ -117,7 +118,11 @@ pub fn find_bottleneck_from_run( ) -> String { let saturations = cluster_saturations_from_run(run_dir, network_bytes_per_sec, io_tps); let bottleneck_name = most_saturated_cluster(&saturations, excluded_names); - println!("Bottleneck from run {}: {}", run_dir.display(), bottleneck_name); + println!( + "Bottleneck from run {}: {}", + run_dir.display(), + bottleneck_name + ); bottleneck_name } @@ -641,7 +646,7 @@ impl NetworkCostTable { /// Total resource cost for sending `count` messages of `message_size_bytes` each. pub fn network_cost(&self, count: usize, message_size_bytes: u64) -> SarStats { if count == 0 || message_size_bytes == 0 { - return SarStats::default() + return SarStats::default(); } self.cost_per_message(message_size_bytes) .scale(count as f64) @@ -999,7 +1004,11 @@ impl SarStats { } pub fn is_zero(&self) -> bool { - self.cpu == 0.0 && self.cpu_user == 0.0 && self.network == 0.0 && self.memory == 0.0 && self.io == 0.0 + self.cpu == 0.0 + && self.cpu_user == 0.0 + && self.network == 0.0 + && self.memory == 0.0 + && self.io == 0.0 } } diff --git a/hydro_optimize/src/partition_syn_analysis.rs b/hydro_optimize/src/partition_syn_analysis.rs index 48ab267..4b9a7b2 100644 --- a/hydro_optimize/src/partition_syn_analysis.rs +++ b/hydro_optimize/src/partition_syn_analysis.rs @@ -184,10 +184,7 @@ impl StructOrTuple { } /// Create a tuple representing dependencies present in both tuples, keeping the more specific dependency if there is one - pub fn intersect( - tuple1: &StructOrTuple, - tuple2: &StructOrTuple, - ) -> Option { + pub fn intersect(tuple1: &StructOrTuple, tuple2: &StructOrTuple) -> Option { // If either tuple1 or tuple2 are empty and None, just return the other tuple for (tuple, other) in [(tuple1, tuple2), (tuple2, tuple1)] { if tuple.is_empty() && tuple.could_be_none { @@ -384,10 +381,7 @@ impl StructOrTuple { /// /// The parent's dependencies are absolute (dependency on an input to the node); /// the child's dependencies are relative (dependency within the function). - pub fn project_parent( - parent: &StructOrTuple, - child: &StructOrTuple, - ) -> Option { + pub fn project_parent(parent: &StructOrTuple, child: &StructOrTuple) -> Option { let mut new_child = StructOrTuple::default(); assert!( !parent.could_be_none && !child.could_be_none, diff --git a/hydro_optimize/src/rewrites.rs b/hydro_optimize/src/rewrites.rs index 5a28201..fb5473c 100644 --- a/hydro_optimize/src/rewrites.rs +++ b/hydro_optimize/src/rewrites.rs @@ -33,7 +33,9 @@ impl VisitMut for ClusterSelfIdReplace { let orig_prefix = format!("__hydro_lang_cluster_self_id_{}", self.orig_cluster_id); let new_prefix = format!("__hydro_lang_cluster_self_id_{}", self.new_cluster_id); if ident.starts_with(&orig_prefix) || ident.starts_with(&new_prefix) { - if ident.starts_with(&orig_prefix) && self.orig_cluster_id != self.new_cluster_id { + if ident.starts_with(&orig_prefix) + && self.orig_cluster_id != self.new_cluster_id + { segment.ident = syn::Ident::new( &format!("__hydro_lang_cluster_self_id_{}", self.new_cluster_id), segment.ident.span(), @@ -277,7 +279,6 @@ pub fn nodes_dependent_on_inputs( dependent_nodes.take() } - /// Check if the type is serializable. Currently a janky implementation that just looks for common unserializable types. /// Add to the list as new errors emerge. fn type_is_serializable(t: &DebugType) -> bool { diff --git a/hydro_optimize_examples/examples/cas.rs b/hydro_optimize_examples/examples/cas.rs index 52968cb..53e6306 100644 --- a/hydro_optimize_examples/examples/cas.rs +++ b/hydro_optimize_examples/examples/cas.rs @@ -28,8 +28,8 @@ struct Args { optimize: bool, } -const WRITE_RATIOS: &[usize] = &[0, 100]; -// const WRITE_RATIOS: &[usize] = &[100, 0]; +// const WRITE_RATIOS: &[usize] = &[0, 100]; +const WRITE_RATIOS: &[usize] = &[100, 0]; /// Compiles the CAS program for a given write ratio. All write ratios produce an identical IR /// (the ratio is a runtime constant inside the client), so they share one optimization run. diff --git a/hydro_optimize_examples/examples/network_calibrator.rs b/hydro_optimize_examples/examples/network_calibrator.rs index 2e14867..12dda1a 100644 --- a/hydro_optimize_examples/examples/network_calibrator.rs +++ b/hydro_optimize_examples/examples/network_calibrator.rs @@ -72,8 +72,8 @@ async fn main() { let clusters = ReusableClusters::default().with_cluster(clients, 1); let processes = ReusableProcesses::default().with_process(server); - let program = - CompiledProgram::new(clusters, processes, location_id_to_cluster).excluding(client_id); + let program = CompiledProgram::new(clusters, processes, location_id_to_cluster) + .excluding(client_id); (builder, program) },