diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index fa3b6a9fed89..e429f4d4ca88 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "22446b7b142461d94c2d08c9fe4db5d2d9047c00777054093d2d528ed771a456", + "checksum": "8a3d7ef6da26d7dfd4b16144813599eb5c196373b3aa8c3c9ce5fdd998cd6417", "crates": { "actix-codec 0.5.1": { "name": "actix-codec", @@ -6378,14 +6378,14 @@ ], "license_file": "LICENSE-APACHE" }, - "backon 0.4.1": { + "backon 1.6.0": { "name": "backon", - "version": "0.4.1", - "package_url": null, + "version": "1.6.0", + "package_url": "https://github.com/Xuanwo/backon", "repository": { "Http": { - "url": "https://static.crates.io/crates/backon/0.4.1/download", - "sha256": "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" + "url": "https://static.crates.io/crates/backon/1.6.0/download", + "sha256": "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" } }, "targets": [ @@ -6407,29 +6407,24 @@ "compile_data_glob": [ "**" ], + "crate_features": { + "common": [ + "std", + "std-blocking-sleep" + ], + "selects": {} + }, "deps": { "common": [ { - "id": "fastrand 1.9.0", + "id": "fastrand 2.3.0", "target": "fastrand" - }, - { - "id": "futures-core 0.3.31", - "target": "futures_core" - }, - { - "id": "pin-project 1.1.2", - "target": "pin_project" - }, - { - "id": "tokio 1.50.0", - "target": "tokio" } ], "selects": {} }, - "edition": "2021", - "version": "0.4.1" + "edition": "2024", + "version": "1.6.0" }, "license": "Apache-2.0", "license_ids": [ @@ -21014,11 +21009,7 @@ "target": "axum_server" }, { - "id": "backoff 0.4.0", - "target": "backoff" - }, - { - "id": "backon 0.4.1", + "id": "backon 1.6.0", "target": "backon" }, { @@ -26270,56 +26261,6 @@ ], "license_file": "LICENSE" }, - "fastrand 1.9.0": { - "name": "fastrand", - "version": "1.9.0", - "package_url": "https://github.com/smol-rs/fastrand", - "repository": { - "Http": { - "url": "https://static.crates.io/crates/fastrand/1.9.0/download", - "sha256": "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" - } - }, - "targets": [ - { - "Library": { - "crate_name": "fastrand", - "crate_root": "src/lib.rs", - "srcs": { - "allow_empty": true, - "include": [ - "**/*.rs" - ] - } - } - } - ], - "library_target_name": "fastrand", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "deps": { - "common": [], - "selects": { - "cfg(all(target_arch = \"wasm32\", not(target_os = \"wasi\")))": [ - { - "id": "instant 0.1.12", - "target": "instant" - } - ] - } - }, - "edition": "2018", - "version": "1.9.0" - }, - "license": "Apache-2.0 OR MIT", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "license_file": "LICENSE-APACHE" - }, "fastrand 2.3.0": { "name": "fastrand", "version": "2.3.0", @@ -98503,8 +98444,7 @@ "axum 0.8.9", "axum-extra 0.10.1", "axum-server 0.7.3", - "backoff 0.4.0", - "backon 0.4.1", + "backon 1.6.0", "base32 0.4.0", "base64 0.13.1", "bech32 0.9.1", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 2c7f4addd97a..1d97decccf15 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -1138,14 +1138,11 @@ dependencies = [ [[package]] name = "backon" -version = "0.4.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "fastrand 1.9.0", - "futures-core", - "pin-project", - "tokio", + "fastrand", ] [[package]] @@ -3532,7 +3529,6 @@ dependencies = [ "axum 0.8.9", "axum-extra 0.10.1", "axum-server 0.7.3", - "backoff", "backon", "base32", "base64 0.13.1", @@ -4519,15 +4515,6 @@ dependencies = [ "regex-syntax 0.8.5", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -4841,7 +4828,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c1155db57329dca6d018b61e76b1488ce9a2e5e44028cac420a5898f4fcef63" dependencies = [ - "fastrand 2.3.0", + "fastrand", "futures-core", "futures-io", "memchr", @@ -9780,7 +9767,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" dependencies = [ "atomic-waker", - "fastrand 2.3.0", + "fastrand", "futures-io", ] @@ -13264,7 +13251,7 @@ version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ - "fastrand 2.3.0", + "fastrand", "getrandom 0.3.4", "once_cell", "rustix 1.1.4", diff --git a/Cargo.lock b/Cargo.lock index 62845f0fe77e..28653c8b98f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1089,6 +1089,15 @@ dependencies = [ "rand 0.8.6", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -1984,7 +1993,7 @@ dependencies = [ name = "canister-test" version = "0.9.0" dependencies = [ - "backoff", + "backon", "candid", "cargo_metadata", "dfn_candid", @@ -6331,7 +6340,7 @@ version = "0.9.0" dependencies = [ "assert_matches", "axum", - "backoff", + "backon", "bytes", "criterion", "futures", @@ -6985,7 +6994,7 @@ dependencies = [ name = "ic-canister-client" version = "0.9.0" dependencies = [ - "backoff", + "backon", "futures-util", "hex", "http-body-util", @@ -7931,7 +7940,7 @@ name = "ic-consensus-manager" version = "0.9.0" dependencies = [ "axum", - "backoff", + "backon", "bytes", "futures", "ic-base-types", @@ -19230,7 +19239,7 @@ dependencies = [ "async-stream", "async-trait", "attestation", - "backoff", + "backon", "candid", "clap", "config_tool", @@ -19883,7 +19892,7 @@ dependencies = [ name = "pocket-ic" version = "15.0.0" dependencies = [ - "backoff", + "backon", "base64 0.13.1", "bitcoin", "candid", @@ -19951,7 +19960,7 @@ dependencies = [ "axum", "axum-extra 0.10.3", "axum-server 0.7.3", - "backoff", + "backon", "base64 0.13.1", "bitcoin-dogecoin", "bytes", @@ -22317,7 +22326,7 @@ name = "sdk-system-tests" version = "0.9.0" dependencies = [ "anyhow", - "backoff", + "backon", "candid", "ic-registry-subnet-type", "ic-system-test-driver", diff --git a/Cargo.toml b/Cargo.toml index c7fd2f1ca9d4..664217ec0d4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -612,7 +612,7 @@ async-trait = "0.1.89" axum = { version = "0.8.4", features = ["ws"] } axum-extra = { version = "0.10.1", features = ["typed-header"] } axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"] } -backoff = "0.4" +backon = { version = "1.6.0", default-features = false, features = ["std"] } base64 = { version = "0.13.1" } bech32 = "^0.9.0" bincode = "1.3.3" diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index 4bb54f075189..0d45a240f5e4 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -143,12 +143,13 @@ crate.spec( version = "^0.7.3", ) crate.spec( - package = "backoff", - version = "^0.4.0", -) -crate.spec( + default_features = False, + features = [ + "std", + "std-blocking-sleep", + ], package = "backon", - version = "^0.4.1", + version = "^1.6.0", ) crate.spec( package = "base32", diff --git a/packages/pocket-ic/BUILD.bazel b/packages/pocket-ic/BUILD.bazel index b1b8acd98c58..4874bbb0a8b9 100644 --- a/packages/pocket-ic/BUILD.bazel +++ b/packages/pocket-ic/BUILD.bazel @@ -12,7 +12,7 @@ rust_library( version = "15.0.0", deps = [ # Keep sorted. - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:base64", "@crate_index//:candid", "@crate_index//:flate2", diff --git a/packages/pocket-ic/Cargo.toml b/packages/pocket-ic/Cargo.toml index ba14e015298e..ef455179503f 100644 --- a/packages/pocket-ic/Cargo.toml +++ b/packages/pocket-ic/Cargo.toml @@ -23,7 +23,7 @@ edition.workspace = true head_nns = [] [dependencies] -backoff = { workspace = true } +backon = { workspace = true } base64 = { workspace = true } candid = { workspace = true } flate2 = { workspace = true } diff --git a/packages/pocket-ic/src/nonblocking.rs b/packages/pocket-ic/src/nonblocking.rs index 3baf36f9beb6..a6bc1960437f 100644 --- a/packages/pocket-ic/src/nonblocking.rs +++ b/packages/pocket-ic/src/nonblocking.rs @@ -16,8 +16,7 @@ use crate::{ CreateCanisterParams, CreateCanisterPlacement, IngressStatusResult, PocketIcBuilder, PocketIcState, RejectResponse, StartServerParams, TickConfigs, Time, copy_dir, start_server, }; -use backoff::backoff::Backoff; -use backoff::{ExponentialBackoff, ExponentialBackoffBuilder}; +use backon::{BackoffBuilder, ExponentialBuilder}; use candid::{ Principal, decode_args, encode_args, utils::{ArgumentDecoder, ArgumentEncoder}, @@ -823,16 +822,19 @@ impl PocketIc { &self, message_id: RawMessageId, ) -> Result, RejectResponse> { - let mut retry_policy: ExponentialBackoff = ExponentialBackoffBuilder::new() - .with_initial_interval(Duration::from_millis(10)) - .with_max_interval(Duration::from_secs(1)) - .with_multiplier(2.0) + let mut retry_policy = ExponentialBuilder::new() + .with_min_delay(Duration::from_millis(10)) + .with_max_delay(Duration::from_secs(1)) + .with_factor(2.0) + .with_jitter() + .with_total_delay(Some(Duration::from_secs(15 * 60))) + .without_max_times() .build(); loop { if let Some(ingress_status) = self.ingress_status(message_id.clone()).await { break ingress_status; } - tokio::time::sleep(retry_policy.next_backoff().unwrap()).await; + tokio::time::sleep(retry_policy.next().unwrap()).await; } } diff --git a/rs/canister_client/BUILD.bazel b/rs/canister_client/BUILD.bazel index 0ad41e7d20ed..7474281fa596 100644 --- a/rs/canister_client/BUILD.bazel +++ b/rs/canister_client/BUILD.bazel @@ -30,7 +30,7 @@ rust_library( "//rs/protobuf", "//rs/types/management_canister_types", "//rs/types/types", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:futures-util", "@crate_index//:http-body-util", "@crate_index//:hyper", @@ -65,7 +65,7 @@ rust_test( "//rs/types/management_canister_types", "//rs/types/types", "//rs/validator", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:futures-util", "@crate_index//:hex", "@crate_index//:http-body-util", @@ -103,7 +103,7 @@ rust_test( "//rs/tree_deserializer", "//rs/types/management_canister_types", "//rs/types/types", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:futures-util", "@crate_index//:hex", "@crate_index//:http-body-util", diff --git a/rs/canister_client/Cargo.toml b/rs/canister_client/Cargo.toml index 1dfd761ad595..1a2593358c00 100644 --- a/rs/canister_client/Cargo.toml +++ b/rs/canister_client/Cargo.toml @@ -7,7 +7,7 @@ description.workspace = true documentation.workspace = true [dependencies] -backoff = { workspace = true } +backon = { workspace = true } futures-util = { workspace = true } hyper = { workspace = true } hyper-rustls = { workspace = true } diff --git a/rs/canister_client/src/agent.rs b/rs/canister_client/src/agent.rs index c3632eb94368..eb9a2893c94a 100644 --- a/rs/canister_client/src/agent.rs +++ b/rs/canister_client/src/agent.rs @@ -3,7 +3,7 @@ use crate::{ cbor::{parse_query_response, prepare_query, prepare_read_state, prepare_update}, http_client::{HttpClient, HttpClientConfig}, }; -use backoff::backoff::Backoff; +use backon::{BackoffBuilder, ExponentialBuilder}; use ic_canister_client_sender::Sender; use ic_crypto_tree_hash::Path; use ic_management_canister_types_private::{IC_00, InstallCodeArgs, Method, Payload}; @@ -54,17 +54,14 @@ pub fn update_path(cid: CanisterId) -> String { const NODE_STATUS_PATH: &str = "api/v2/status"; const CATCH_UP_PACKAGE_PATH: &str = "/_/catch_up_package"; -pub fn get_backoff_policy() -> backoff::ExponentialBackoff { - backoff::ExponentialBackoff { - initial_interval: MIN_POLL_INTERVAL, - current_interval: MIN_POLL_INTERVAL, - randomization_factor: 0.1, - multiplier: POLL_INTERVAL_MULTIPLIER, - start_time: std::time::Instant::now(), - max_interval: MAX_POLL_INTERVAL, - max_elapsed_time: None, - clock: backoff::SystemClock::default(), - } +pub fn get_backoff_policy() -> backon::ExponentialBackoff { + ExponentialBuilder::new() + .with_min_delay(MIN_POLL_INTERVAL) + .with_max_delay(MAX_POLL_INTERVAL) + .with_factor(POLL_INTERVAL_MULTIPLIER as f32) + .with_jitter() + .without_max_times() + .build() } /// An agent to talk to the Internet Computer through the public endpoints. @@ -281,7 +278,7 @@ impl Agent { // will take at least the time between consensus blocks. while next_poll_time < deadline { sleep_until(tokio::time::Instant::from_std(next_poll_time)).await; - next_poll_time = Instant::now() + backoff.next_backoff().expect("Backoff interval MUST be available. If you see this error the backoff is misconfigured."); + next_poll_time = Instant::now() + backoff.next().expect("Backoff interval MUST be available. If you see this error the backoff is misconfigured."); match self .wait_ingress(request_id.clone(), deadline, effective_canister_id) .await diff --git a/rs/orchestrator/BUILD.bazel b/rs/orchestrator/BUILD.bazel index b11c9dc37148..06df34682a00 100644 --- a/rs/orchestrator/BUILD.bazel +++ b/rs/orchestrator/BUILD.bazel @@ -52,7 +52,7 @@ rust_library( "//rs/types/management_canister_types", "//rs/types/types", "@crate_index//:anyhow", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:candid", "@crate_index//:clap", "@crate_index//:der", diff --git a/rs/orchestrator/Cargo.toml b/rs/orchestrator/Cargo.toml index 0a9d73304e21..4fa2b992e93d 100644 --- a/rs/orchestrator/Cargo.toml +++ b/rs/orchestrator/Cargo.toml @@ -10,7 +10,7 @@ documentation.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } attestation = { path = "../ic_os/sev/attestation" } -backoff = { workspace = true } +backon = { workspace = true } candid = { workspace = true } clap = { workspace = true } config_tool = { path = "../ic_os/config/tool" } diff --git a/rs/orchestrator/src/hostos_upgrade.rs b/rs/orchestrator/src/hostos_upgrade.rs index c0707cefbb6e..6a0b88d327a5 100644 --- a/rs/orchestrator/src/hostos_upgrade.rs +++ b/rs/orchestrator/src/hostos_upgrade.rs @@ -2,7 +2,7 @@ use crate::{ error::{OrchestratorError, OrchestratorResult}, registry_helper::RegistryHelper, }; -use backoff::{ExponentialBackoff, backoff::Backoff}; +use backon::{BackoffBuilder, ExponentialBuilder}; use ic_logger::{ReplicaLogger, info, warn}; use ic_protobuf::registry::hostos_version::v1::HostosVersionRecord; use ic_sys::utility_command::UtilityCommand; @@ -40,22 +40,24 @@ impl HostosUpgrader { pub async fn upgrade_loop( &mut self, cancellation_token: CancellationToken, - mut backoff: ExponentialBackoff, + backoff_builder: ExponentialBuilder, + max_interval: Duration, liveness_timeout: Duration, ) { + let mut backoff = backoff_builder.build(); loop { match tokio::time::timeout(liveness_timeout, self.check_for_upgrade()).await { - Ok(Ok(())) => backoff.reset(), + Ok(Ok(())) => backoff = backoff_builder.build(), e => warn!(&self.logger, "Check for HostOS upgrade failed: {:?}", e), } - // NOTE: We currently do not and should not set `max_elapsed_time`, - // so that we never run out of backoffs. If `max_elapsed_time` _is_ - // ever set, repeat the `max_interval` instead. This is technically - // not the same behavior as if `max_elapsed_time` was unset, because - // we will not be including jitter, but it should be close enough, - // and safe. - let safe_backoff = backoff.next_backoff().unwrap_or(backoff.max_interval); + // NOTE: We currently do not and should not set a total delay on + // `backoff_builder`, so that we never run out of backoffs. If a + // total delay _is_ ever set, repeat `max_interval` instead. This + // is technically not the same behavior as if no total delay was + // set, because we will not be including jitter, but it should be + // close enough, and safe. + let safe_backoff = backoff.next().unwrap_or(max_interval); tokio::select! { _ = tokio::time::sleep(safe_backoff) => {} _ = cancellation_token.cancelled() => break diff --git a/rs/orchestrator/src/orchestrator.rs b/rs/orchestrator/src/orchestrator.rs index 54cba30a9217..1e3f54b6f46d 100644 --- a/rs/orchestrator/src/orchestrator.rs +++ b/rs/orchestrator/src/orchestrator.rs @@ -17,7 +17,7 @@ use crate::{ upgrade::{OrchestratorControlFlow, Upgrade}, }; use anyhow::Context; -use backoff::ExponentialBackoffBuilder; +use backon::ExponentialBuilder; use config_tool::guestos::network::{get_best_interface_name, get_interface_addresses}; use guest_upgrade_server::orchestrator::new_disk_encryption_key_exchange_server_agent_for_orchestrator; use ic_config::{ @@ -534,17 +534,22 @@ impl Orchestrator { // e.g. (roughly) 1, 1.75, 3, 5.25, 9.5, 16.5, 28.75, 50.25, 88, 120, 120 // // Additionally, there's a random +=50% range added to each delay, for jitter. - let backoff = ExponentialBackoffBuilder::new() - .with_initial_interval(Duration::from_secs(60)) - .with_randomization_factor(0.5) - .with_multiplier(1.75) - .with_max_interval(Duration::from_secs(2 * 60 * 60)) - .with_max_elapsed_time(None) - .build(); + let max_interval = Duration::from_secs(2 * 60 * 60); + let backoff_builder = ExponentialBuilder::new() + .with_min_delay(Duration::from_secs(60)) + .with_jitter() + .with_factor(1.75) + .with_max_delay(max_interval) + .without_max_times(); let liveness_timeout = Duration::from_secs(15 * 60); upgrade - .upgrade_loop(cancellation_token, backoff, liveness_timeout) + .upgrade_loop( + cancellation_token, + backoff_builder, + max_interval, + liveness_timeout, + ) .await; } diff --git a/rs/p2p/artifact_downloader/BUILD.bazel b/rs/p2p/artifact_downloader/BUILD.bazel index bf9235084209..c812a70f45fb 100644 --- a/rs/p2p/artifact_downloader/BUILD.bazel +++ b/rs/p2p/artifact_downloader/BUILD.bazel @@ -20,7 +20,7 @@ rust_library( "//rs/types/base_types", "//rs/types/types", "@crate_index//:axum", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:bytes", "@crate_index//:prometheus", "@crate_index//:rand", @@ -52,7 +52,7 @@ rust_test( "//rs/types/types_test_utils", "@crate_index//:assert_matches", "@crate_index//:axum", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:bytes", "@crate_index//:futures", "@crate_index//:http-body-util", @@ -87,7 +87,7 @@ rust_test_suite( "//rs/types/types_test_utils", "@crate_index//:assert_matches", "@crate_index//:axum", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:bytes", "@crate_index//:futures", "@crate_index//:http-body-util", diff --git a/rs/p2p/artifact_downloader/Cargo.toml b/rs/p2p/artifact_downloader/Cargo.toml index 9164573d532e..423516c93acf 100644 --- a/rs/p2p/artifact_downloader/Cargo.toml +++ b/rs/p2p/artifact_downloader/Cargo.toml @@ -10,7 +10,7 @@ documentation.workspace = true [dependencies] axum = { workspace = true } -backoff = { workspace = true } +backon = { workspace = true } bytes = { workspace = true } ic-base-types = { path = "../../types/base_types" } ic-interfaces = { path = "../../interfaces" } diff --git a/rs/p2p/artifact_downloader/src/fetch_artifact/download.rs b/rs/p2p/artifact_downloader/src/fetch_artifact/download.rs index c9736256611d..21c29a6c1917 100644 --- a/rs/p2p/artifact_downloader/src/fetch_artifact/download.rs +++ b/rs/p2p/artifact_downloader/src/fetch_artifact/download.rs @@ -9,7 +9,7 @@ use axum::{ http::{Request, StatusCode}, routing::any, }; -use backoff::{ExponentialBackoffBuilder, backoff::Backoff}; +use backon::{BackoffBuilder, ExponentialBuilder}; use bytes::Bytes; use ic_base_types::NodeId; use ic_interfaces::p2p::consensus::{ @@ -213,10 +213,12 @@ impl FetchArtifact { return AssembleResult::Unwanted; } - let mut artifact_download_backoff = ExponentialBackoffBuilder::new() - .with_initial_interval(MIN_ARTIFACT_RPC_TIMEOUT) - .with_max_interval(MAX_ARTIFACT_RPC_TIMEOUT) - .with_max_elapsed_time(None) + let mut artifact_download_backoff = ExponentialBuilder::new() + .with_min_delay(MIN_ARTIFACT_RPC_TIMEOUT) + .with_max_delay(MAX_ARTIFACT_RPC_TIMEOUT) + .with_factor(1.5) + .with_jitter() + .without_max_times() .build(); match artifact { @@ -237,7 +239,7 @@ impl FetchArtifact { let result = loop { let next_request_at = Instant::now() + artifact_download_backoff - .next_backoff() + .next() .unwrap_or(MAX_ARTIFACT_RPC_TIMEOUT); if let Some(peer) = peer_rx.peers().into_iter().choose(&mut rng) { let bytes = Bytes::from(Artifact::PbId::proxy_encode(id.clone())); diff --git a/rs/p2p/artifact_downloader/src/fetch_stripped_artifact/download.rs b/rs/p2p/artifact_downloader/src/fetch_stripped_artifact/download.rs index 49b4483310bc..5c57722d2ee4 100644 --- a/rs/p2p/artifact_downloader/src/fetch_stripped_artifact/download.rs +++ b/rs/p2p/artifact_downloader/src/fetch_stripped_artifact/download.rs @@ -9,7 +9,7 @@ use axum::{ http::{Request, StatusCode}, routing::any, }; -use backoff::{ExponentialBackoffBuilder, backoff::Backoff}; +use backon::{BackoffBuilder, ExponentialBuilder}; use bytes::Bytes; use ic_interfaces::p2p::consensus::{Peers, ValidatedPoolReader}; use ic_logger::{ReplicaLogger, warn}; @@ -298,10 +298,12 @@ pub(crate) async fn download_stripped_message( ) -> (StrippedMessage, NodeId) { let message_type = StrippedMessageType::from(&stripped_message_id); metrics.report_started_stripped_message_download(message_type); - let mut artifact_download_timeout = ExponentialBackoffBuilder::new() - .with_initial_interval(MIN_ARTIFACT_RPC_TIMEOUT) - .with_max_interval(MAX_ARTIFACT_RPC_TIMEOUT) - .with_max_elapsed_time(None) + let mut artifact_download_timeout = ExponentialBuilder::new() + .with_min_delay(MIN_ARTIFACT_RPC_TIMEOUT) + .with_max_delay(MAX_ARTIFACT_RPC_TIMEOUT) + .with_factor(1.5) + .with_jitter() + .without_max_times() .build(); let mut rng = SmallRng::from_entropy(); @@ -332,7 +334,7 @@ pub(crate) async fn download_stripped_message( loop { let next_request_at = Instant::now() + artifact_download_timeout - .next_backoff() + .next() .unwrap_or(MAX_ARTIFACT_RPC_TIMEOUT); if let Some(peer) = { peer_rx.peers().into_iter().choose(&mut rng) } { match timeout_at(next_request_at, transport.rpc(&peer, request.clone())).await { diff --git a/rs/p2p/consensus_manager/BUILD.bazel b/rs/p2p/consensus_manager/BUILD.bazel index 8a3207ae8d89..f8dd1483a162 100644 --- a/rs/p2p/consensus_manager/BUILD.bazel +++ b/rs/p2p/consensus_manager/BUILD.bazel @@ -22,7 +22,7 @@ rust_library( "//rs/types/base_types", "//rs/types/types", "@crate_index//:axum", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:bytes", "@crate_index//:prometheus", "@crate_index//:prost", @@ -54,7 +54,7 @@ rust_test( "//rs/types/types", "//rs/types/types_test_utils", "@crate_index//:axum", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:bytes", "@crate_index//:futures", "@crate_index//:mockall", @@ -89,7 +89,7 @@ rust_test_suite( "//rs/types/types", "//rs/types/types_test_utils", "@crate_index//:axum", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:bytes", "@crate_index//:futures", "@crate_index//:mockall", diff --git a/rs/p2p/consensus_manager/Cargo.toml b/rs/p2p/consensus_manager/Cargo.toml index d3ed45295bcf..20b2b4c70c20 100644 --- a/rs/p2p/consensus_manager/Cargo.toml +++ b/rs/p2p/consensus_manager/Cargo.toml @@ -10,7 +10,7 @@ documentation.workspace = true [dependencies] axum = { workspace = true } -backoff = { workspace = true } +backon = { workspace = true } bytes = { workspace = true } ic-base-types = { path = "../../types/base_types" } ic-interfaces = { path = "../../interfaces" } diff --git a/rs/p2p/consensus_manager/src/sender.rs b/rs/p2p/consensus_manager/src/sender.rs index 7bf857c7ad1d..79275522b9af 100644 --- a/rs/p2p/consensus_manager/src/sender.rs +++ b/rs/p2p/consensus_manager/src/sender.rs @@ -7,7 +7,7 @@ use std::{ }; use axum::http::Request; -use backoff::{ExponentialBackoffBuilder, backoff::Backoff}; +use backon::{BackoffBuilder, ExponentialBuilder}; use bytes::Bytes; use ic_base_types::NodeId; use ic_interfaces::p2p::consensus::{ArtifactAssembler, ArtifactTransmit, ArtifactWithOpt}; @@ -311,11 +311,12 @@ async fn send_transmit_to_peer( peer: NodeId, route: String, ) { - let mut backoff = ExponentialBackoffBuilder::new() - .with_initial_interval(MIN_BACKOFF_INTERVAL) - .with_max_interval(MAX_BACKOFF_INTERVAL) - .with_multiplier(BACKOFF_MULTIPLIER) - .with_max_elapsed_time(None) + let mut backoff = ExponentialBuilder::new() + .with_min_delay(MIN_BACKOFF_INTERVAL) + .with_max_delay(MAX_BACKOFF_INTERVAL) + .with_factor(BACKOFF_MULTIPLIER as f32) + .with_jitter() + .without_max_times() .build(); loop { @@ -329,7 +330,7 @@ async fn send_transmit_to_peer( return; } - let backoff_duration = backoff.next_backoff().unwrap_or(MAX_BACKOFF_INTERVAL); + let backoff_duration = backoff.next().unwrap_or(MAX_BACKOFF_INTERVAL); time::sleep(backoff_duration).await; } } diff --git a/rs/pocket_ic_server/BUILD.bazel b/rs/pocket_ic_server/BUILD.bazel index e55c82fab9f2..21c883c139d0 100644 --- a/rs/pocket_ic_server/BUILD.bazel +++ b/rs/pocket_ic_server/BUILD.bazel @@ -62,7 +62,7 @@ LIB_DEPENDENCIES = [ "@crate_index//:axum", "@crate_index//:axum-extra", "@crate_index//:axum-server", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:base64", "@crate_index//:bitcoin_dogecoin", "@crate_index//:bytes", diff --git a/rs/pocket_ic_server/Cargo.toml b/rs/pocket_ic_server/Cargo.toml index 3ffad6ef62a1..3220fc30004a 100644 --- a/rs/pocket_ic_server/Cargo.toml +++ b/rs/pocket_ic_server/Cargo.toml @@ -10,7 +10,7 @@ async-trait = { workspace = true } axum = { workspace = true } axum-extra = { workspace = true } axum-server = { workspace = true } -backoff = { workspace = true } +backon = { workspace = true } base64 = { workspace = true } bitcoin = { workspace = true } bytes = { workspace = true } diff --git a/rs/pocket_ic_server/src/state_api/routes.rs b/rs/pocket_ic_server/src/state_api/routes.rs index 16c02b59dcde..38d28386e9ef 100644 --- a/rs/pocket_ic_server/src/state_api/routes.rs +++ b/rs/pocket_ic_server/src/state_api/routes.rs @@ -35,8 +35,7 @@ use axum::{ }; use axum_extra::headers; use axum_extra::headers::HeaderMapExt; -use backoff::backoff::Backoff; -use backoff::{ExponentialBackoff, ExponentialBackoffBuilder}; +use backon::{BackoffBuilder, ExponentialBuilder}; use ic_boundary::{ErrorClientFacing, MAX_REQUEST_BODY_SIZE}; use ic_http_endpoints_public::{cors_layer, make_plaintext_response, query, read_state}; use ic_registry_routing_table::RoutingTable; @@ -321,11 +320,13 @@ async fn run_operation( ) -> (StatusCode, ApiResponse) { let retry_if_busy = op.retry_if_busy(); let op = Arc::new(op); - let mut retry_policy: ExponentialBackoff = ExponentialBackoffBuilder::new() - .with_initial_interval(Duration::from_millis(10)) - .with_max_interval(Duration::from_secs(1)) - .with_multiplier(2.0) - .with_max_elapsed_time(Some(Duration::from_secs(RETRY_TIMEOUT_S))) + let mut retry_policy = ExponentialBuilder::new() + .with_min_delay(Duration::from_millis(10)) + .with_max_delay(Duration::from_secs(1)) + .with_factor(2.0) + .with_jitter() + .with_total_delay(Some(Duration::from_secs(RETRY_TIMEOUT_S))) + .without_max_times() .build(); loop { match api_state @@ -359,7 +360,7 @@ async fn run_operation( "run_operation::retry_busy instance_id={} state_label={:?} op_id={}", instance_id, state_label, op_id.0 ); - match retry_policy.next_backoff() { + match retry_policy.next() { Some(duration) => tokio::time::sleep(duration).await, None => { break ( diff --git a/rs/rust_canisters/canister_test/BUILD.bazel b/rs/rust_canisters/canister_test/BUILD.bazel index ca259491cf2b..002547883c8c 100644 --- a/rs/rust_canisters/canister_test/BUILD.bazel +++ b/rs/rust_canisters/canister_test/BUILD.bazel @@ -21,7 +21,7 @@ rust_library( "//rs/types/cycles", "//rs/types/management_canister_types", "//rs/types/types", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:candid", "@crate_index//:cargo_metadata", "@crate_index//:escargot", diff --git a/rs/rust_canisters/canister_test/Cargo.toml b/rs/rust_canisters/canister_test/Cargo.toml index 19c2bc8d9ac5..e82ccecdb3eb 100644 --- a/rs/rust_canisters/canister_test/Cargo.toml +++ b/rs/rust_canisters/canister_test/Cargo.toml @@ -7,7 +7,7 @@ description.workspace = true documentation.workspace = true [dependencies] -backoff = { workspace = true } +backon = { workspace = true } candid = { workspace = true } cargo_metadata = { workspace = true } dfn_candid = { path = "../dfn_candid" } diff --git a/rs/rust_canisters/canister_test/src/canister.rs b/rs/rust_canisters/canister_test/src/canister.rs index f233e66b7e7b..e5c6fcb0354f 100644 --- a/rs/rust_canisters/canister_test/src/canister.rs +++ b/rs/rust_canisters/canister_test/src/canister.rs @@ -1,4 +1,4 @@ -use backoff::backoff::Backoff; +use backon::{BackoffBuilder, ExponentialBuilder}; use candid::Principal; use core::future::Future; use dfn_candid::{candid, candid_multi_arity, candid_one}; @@ -42,17 +42,15 @@ const MAX_BACKOFF_INTERVAL: Duration = Duration::from_secs(10); const BACKOFF_INTERVAL_MULTIPLIER: f64 = 1.1; const MAX_ELAPSED_TIME: Duration = Duration::from_secs(60 * 5); // 5 minutes -pub fn get_backoff_policy() -> backoff::ExponentialBackoff { - backoff::ExponentialBackoff { - initial_interval: MIN_BACKOFF_INTERVAL, - current_interval: MIN_BACKOFF_INTERVAL, - randomization_factor: 0.1, - multiplier: BACKOFF_INTERVAL_MULTIPLIER, - start_time: std::time::Instant::now(), - max_interval: MAX_BACKOFF_INTERVAL, - max_elapsed_time: Some(MAX_ELAPSED_TIME), - clock: backoff::SystemClock::default(), - } +pub fn get_backoff_policy() -> backon::ExponentialBackoff { + ExponentialBuilder::new() + .with_min_delay(MIN_BACKOFF_INTERVAL) + .with_max_delay(MAX_BACKOFF_INTERVAL) + .with_factor(BACKOFF_INTERVAL_MULTIPLIER as f32) + .with_jitter() + .with_total_delay(Some(MAX_ELAPSED_TIME)) + .without_max_times() + .build() } #[derive(Clone)] @@ -229,7 +227,7 @@ impl Wasm { eprintln!( "Installation of wasm into canister with ID: {canister_id} failed with: {e}" ); - match backoff.next_backoff() { + match backoff.next() { Some(interval) => { tokio::time::sleep(interval).await; } @@ -258,7 +256,7 @@ where loop { match f().await { Ok(v) => return Ok(v), - Err(e) => match backoff.next_backoff() { + Err(e) => match backoff.next() { Some(interval) => { eprintln!("Retrying due to: {}", e); tokio::time::sleep(interval).await; diff --git a/rs/tests/sdk/BUILD.bazel b/rs/tests/sdk/BUILD.bazel index 86e4fe1114c8..2c9620ccce0f 100644 --- a/rs/tests/sdk/BUILD.bazel +++ b/rs/tests/sdk/BUILD.bazel @@ -23,7 +23,7 @@ rust_library( # Keep sorted. "//rs/tests/driver:ic-system-test-driver", "//rs/types/types", - "@crate_index//:backoff", + "@crate_index//:backon", "@crate_index//:candid", "@crate_index//:reqwest", "@crate_index//:serde_json", diff --git a/rs/tests/sdk/Cargo.toml b/rs/tests/sdk/Cargo.toml index 925bfff5af72..401e0b5d3486 100644 --- a/rs/tests/sdk/Cargo.toml +++ b/rs/tests/sdk/Cargo.toml @@ -8,7 +8,7 @@ documentation.workspace = true [dependencies] anyhow = { workspace = true } -backoff = { workspace = true } +backon = { workspace = true, features = ["std-blocking-sleep"] } candid = { workspace = true } ic-registry-subnet-type = { path = "../../registry/subnet_type" } ic-system-test-driver = { path = "../driver" } diff --git a/rs/tests/sdk/src/asset.rs b/rs/tests/sdk/src/asset.rs index 709437cabe33..40fca81c825e 100644 --- a/rs/tests/sdk/src/asset.rs +++ b/rs/tests/sdk/src/asset.rs @@ -1,4 +1,4 @@ -use backoff::{ExponentialBackoff, retry_notify}; +use backon::{BlockingRetryable, ExponentialBuilder}; use candid::Principal; use ic_system_test_driver::driver::{ic_gateway_vm::HasIcGatewayVm, test_env::TestEnv}; use slog::{error, info}; @@ -29,12 +29,15 @@ pub fn get_asset_as_string( let resolve_override = ic_gateway.resolve_override_for_url(&parsed_asset_url); let accept_invalid_certs = ic_gateway.uses_self_signed_cert(); - let backoff = ExponentialBackoff { - max_elapsed_time: Some(Duration::from_secs(120)), - ..Default::default() - }; + let backoff = ExponentialBuilder::new() + .with_min_delay(Duration::from_millis(500)) + .with_max_delay(Duration::from_secs(60)) + .with_factor(1.5) + .with_jitter() + .with_total_delay(Some(Duration::from_secs(120))) + .without_max_times(); - let notify = |err, dur| { + let notify = |err: &reqwest::Error, dur: Duration| { error!(log, "error: {err}"); error!(log, "retry in {dur:?}"); }; @@ -52,7 +55,7 @@ pub fn get_asset_as_string( Ok(body) }; - let body = retry_notify(backoff, operation, notify).unwrap(); + let body = operation.retry(backoff).notify(notify).call().unwrap(); info!(log, "response body: {body}"); body