feat!: Auto decouple and partition

Cargo.toml

@@ -8,24 +8,24 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-hydro_lang = { version = "0.15.0", git = "https://github.com/hydro-project/hydro.git", rev = "e5099aaa" }
-hydro_std = { version = "0.15.0", git = "https://github.com/hydro-project/hydro.git", rev = "e5099aaa" }
-hydro_test = { version = "0.0.0", git = "https://github.com/hydro-project/hydro.git", rev = "e5099aaa" }
-dfir_lang = { version = "0.15.0", git = "https://github.com/hydro-project/hydro.git", rev = "e5099aaa" }
-hydro_build_utils = { version = "0.0.1", git = "https://github.com/hydro-project/hydro.git", rev = "e5099aaa", features = ["insta"] }
-hydro_deploy = { version = "0.15.0", git = "https://github.com/hydro-project/hydro.git", rev = "e5099aaa", features = ["profile-folding"] }
-include_mdtests = { version = "0.0.0", git = "https://github.com/hydro-project/hydro.git", rev = "e5099aaa" }
+hydro_lang = { version = "0.16.0", git = "https://github.com/davidchuyaya/hydroflow.git", branch = "fix_network_pull_count" }
+hydro_std = { version = "0.16.0", git = "https://github.com/davidchuyaya/hydroflow.git", branch = "fix_network_pull_count" }
+hydro_test = { version = "0.0.0", git = "https://github.com/davidchuyaya/hydroflow.git", branch = "fix_network_pull_count" }
+dfir_lang = { version = "0.16.0", git = "https://github.com/davidchuyaya/hydroflow.git", branch = "fix_network_pull_count" }
+hydro_build_utils = { version = "0.1.0", git = "https://github.com/davidchuyaya/hydroflow.git", branch = "fix_network_pull_count", features = ["insta"] }
+hydro_deploy = { version = "0.16.0", git = "https://github.com/davidchuyaya/hydroflow.git", branch = "fix_network_pull_count", features = ["profile-folding"] }
+include_mdtests = { version = "0.0.0", git = "https://github.com/davidchuyaya/hydroflow.git", branch = "fix_network_pull_count" }
 serde = { version = "1.0.197", features = ["derive"] }
-stageleft = "0.13.4"
-stageleft_tool = "0.13.4"
+stageleft = "0.13.5"
+stageleft_tool = "0.13.5"
 tokio = { version = "1.29.0", features = ["full"] }
 ctor = "0.2"
 clap = { version = "4.4", features = ["derive"] }
 regex = "1.11.1"
 
 
 # Uncomment to use on cloud desktop
-# [patch."https://github.com/hydro-project/hydro.git"]
+# [patch."https://github.com/davidchuyaya/hydroflow.git"]
 # hydro_lang = { path = "/local/home/chudc/hydroflow/hydro_lang" }
 # hydro_std = { path = "/local/home/chudc/hydroflow/hydro_std" }
 # hydro_test = { path = "/local/home/chudc/hydroflow/hydro_test" }
@@ -35,11 +35,11 @@ regex = "1.11.1"
 # include_mdtests = { path = "/local/home/chudc/hydroflow/include_mdtests" }
 
 # Uncomment to use local hydro checkout on Mac
-# [patch."https://github.com/hydro-project/hydro.git"]
-# hydro_lang = { path = "/Users/chudc/Desktop/hydro/hydro_lang" }
-# hydro_std = { path = "/Users/chudc/Desktop/hydro/hydro_std" }
-# hydro_test = { path = "/Users/chudc/Desktop/hydro/hydro_test" }
-# dfir_lang = { path = "/Users/chudc/Desktop/hydro/dfir_lang" }
-# hydro_build_utils = { path = "/Users/chudc/Desktop/hydro/hydro_build_utils" }
-# hydro_deploy = { path = "/Users/chudc/Desktop/hydro/hydro_deploy/core" }
-# include_mdtests = { path = "/Users/chudc/Desktop/hydro/include_mdtests" }
+# [patch."https://github.com/davidchuyaya/hydroflow.git"]
+# hydro_lang = { path = "/home/chudc/hydroflow/hydro_lang" }
+# hydro_std = { path = "/home/chudc/hydroflow/hydro_std" }
+# hydro_test = { path = "/home/chudc/hydroflow/hydro_test" }
+# dfir_lang = { path = "/home/chudc/hydroflow/dfir_lang" }
+# hydro_build_utils = { path = "/home/chudc/hydroflow/hydro_build_utils" }
+# hydro_deploy = { path = "/home/chudc/hydroflow/hydro_deploy/core" }
+# include_mdtests = { path = "/home/chudc/hydroflow/include_mdtests" }

README.md

@@ -1,22 +1,69 @@
 # Hydro Optimize
 
+Automatically apply decoupling and partitioning to Hydro programs for higher throughput.
+
+> ![NOTE]
+> Only Linux is supported, as we compile with `glibc` for better performance and more legibile `perf` results.
+
+
 ## Installation
 
 ### ILP Solver
-These crates rely on the [HiGHS](https://highs.dev/) ILP solver to find the optimal set of rewrites. The cmake compiler is required:
+We rely on the [Gurobi](https://www.gurobi.com/) ILP solver to find the optimal set of rewrites.
+Gurobi is not free; you will need to either create a Gurobi account (and use a free educational license) or use your organization's license, if available.
+
+```bash
+./setup_gurobi.sh
+```
+
+Create a Gurobi license following your organization's instructions.
+For free educational licenses, you may request a named license from Gurobi, then run the command they provide, which should resemble the following:
+
+```bash
+grbgetkey <unique-hash-from-gurobi>
+```
+
+## Execution
+
+To run Microbus or Krupa without applying any optimizations, run:
+```bash
+cargo run --example benchmark_paxos -- --aws > paxos.txt
+cargo run --example cas -- --aws > cas.txt
+```
+The current progress of each execution can be found in the piped `.txt` files.
+Note that killing the process (or putting the running machine to sleep) will leave the machines hanging, so if you must kill the process, be sure to follow instructions in "Tearing Down" to tear it down.
+
+### Automatic Optimization
+
+Optimization relies on an accurate model of networking costs, which must first be calibrated with NetworkCalibrator.
+This may take multiple hours, as it cycles through multiple network sizes, and gradually increases the number of clients for each size until throughput saturates.
+```bash
+cargo run --example network_calibrator -- --aws > network.txt
+```
 
-Linux:
+Once network calibration is complete, we can optimize either protocol by adding `--optimize`. For example:
 ```bash
-sudo apt-get install cmake
+cargo run --example benchmark_paxos -- --aws --optimize > paxos_optimize.txt
 ```
 
-MacOS (assuming HomeBrew is installed):
+Run the command above continuously in a loop until it returns with no outputs.
+Use the following script to automate that process:
 ```bash
-brew install cmake
+./run_protocol.sh benchmark_paxos
+./run_protocol.sh cas
 ```
 
-### Profiling
-MacOS profiling requires [samply](https://github.com/mstange/samply):
+All results are stored in `benchmark_results`.
+
+## Tearing Down
+
+If the experiment ever goes wrong, you can terminate it with `Ctrl+C`.
+Then delete any cloud resources with:
+```bash
+./terraform_cleanup.sh
+```
+
+Afterwards, you may optionally delete the terraform files to speed up future terraform cleanups:
 ```bash
-cargo install --locked samply
-```
+rm -rf .hydro
+```

hydro_optimize/Cargo.toml

@@ -10,9 +10,8 @@ edition = "2024"
 all-features = true
 
 [dependencies]
-chrono = "0.4.44"
 ena = "0.14.4"
-good_lp = { version = "1.14.0", features = ["highs"], default-features = false }
+good_lp = { version = "1.14.0", features = ["lp-solvers"], default-features = false }
 hydro_deploy.workspace = true
 hydro_lang = { workspace = true, features = ["deploy"] }
 hydro_std.workspace = true