diff --git a/bootstrap/bootstrap/Cargo.lock b/bootstrap/bootstrap/Cargo.lock index d8f21893..97ebf9e7 100644 --- a/bootstrap/bootstrap/Cargo.lock +++ b/bootstrap/bootstrap/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "ansi_term" @@ -71,6 +71,7 @@ dependencies = [ "serde_json", "shlex", "simple_logger", + "url", "walkdir", ] @@ -164,6 +165,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", +] + [[package]] name = "encoding_rs" version = "0.8.32" @@ -236,9 +248,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -411,14 +423,111 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "idna" -version = "0.3.0" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", ] [[package]] @@ -473,6 +582,12 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + [[package]] name = "log" version = "0.4.17" @@ -624,9 +739,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" @@ -646,6 +761,15 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -967,6 +1091,12 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "socket2" version = "0.4.7" @@ -977,6 +1107,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.8.0" @@ -1005,6 +1141,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -1062,20 +1209,15 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.6.0" +name = "tinystr" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ - "tinyvec_macros", + "displaydoc", + "zerovec", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.25.0" @@ -1149,27 +1291,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" -[[package]] -name = "unicode-bidi" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" - [[package]] name = "unicode-ident" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-width" version = "0.1.10" @@ -1178,15 +1305,21 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "url" -version = "2.3.1" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1428,3 +1561,87 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", +] diff --git a/bootstrap/bootstrap/Cargo.toml b/bootstrap/bootstrap/Cargo.toml index b6bae5d4..ef3fe17b 100644 --- a/bootstrap/bootstrap/Cargo.toml +++ b/bootstrap/bootstrap/Cargo.toml @@ -21,6 +21,7 @@ openssl-sys = "*" walkdir = "2" clap = "2.33" exec = "0.3" +url = "2.5.4" [features] # Force openssl-sys to staticly link in the openssl library. Necessary when diff --git a/bootstrap/bootstrap/src/bootstrap.rs b/bootstrap/bootstrap/src/bootstrap.rs index bc2e4fbf..6aaebd27 100644 --- a/bootstrap/bootstrap/src/bootstrap.rs +++ b/bootstrap/bootstrap/src/bootstrap.rs @@ -34,7 +34,7 @@ pub fn update(_cfg: &config::Config) -> Result<()> { // the actual system bootstrap. fn update_bootstrap() -> Result<()> { // we are running in a tmpfs workdir in this method - let repo = hub::Repo::new("tf-autobuilder"); + let repo = hub::Repo::new("tf-autobuilder")?; let name = BOOTSTAP_FLIST; let flist = retry::retry(retry::delay::Exponential::from_millis(200), || { @@ -66,7 +66,7 @@ fn update_bootstrap() -> Result<()> { ///install installs all binaries from the tf-zos-bins repo pub fn install(cfg: &config::Config) -> Result<()> { - let repo = Repo::new(ZOS_REPO); + let repo = Repo::new(ZOS_REPO)?; let runmode = cfg.runmode.to_string(); @@ -100,7 +100,7 @@ pub fn install(cfg: &config::Config) -> Result<()> { tag.write(FLIST_TAG_FILE)?; let (repo, tag) = tag.tag_link(); - let client = Repo::new(repo); + let client = Repo::new(repo)?; let packages = client.list_tag(tag)?.context("tag is not found")?; // new style setup, just install every thing. diff --git a/bootstrap/bootstrap/src/hub.rs b/bootstrap/bootstrap/src/hub.rs index e67c8b35..db3566cd 100644 --- a/bootstrap/bootstrap/src/hub.rs +++ b/bootstrap/bootstrap/src/hub.rs @@ -1,15 +1,58 @@ +use crate::config::RunMode; use anyhow::Result; use reqwest::{blocking::get, StatusCode}; +use retry::delay::Exponential; +use retry::{retry, OperationResult}; use serde::{Deserialize, Serialize}; use serde_json; use std::fs::{write, OpenOptions}; use std::io::copy; use std::path::Path; -const HUB: &str = "https://v4.hub.grid.tf"; +#[derive(Deserialize)] +struct ZosConfig { + hub_url: Vec, +} + +fn get_hub_url(runmode: &RunMode) -> Result> { + let base_url = "https://github.com/threefoldtech/zos-config/raw/main/"; + let config_filename = match runmode { + RunMode::Prod => "production.json", + RunMode::Dev => "development.json", + RunMode::Test => "testing.json", + RunMode::QA => "qa.json", + }; + + let config_url = format!("{}/{}", base_url, config_filename); + let fallback = vec!["https://hub.grid.tf".to_string()]; + + let hub_urls = retry(Exponential::from_millis(1000).take(5), || { + match get(config_url.as_str()) { + Ok(resp) if resp.status().is_success() => OperationResult::Ok(resp), + Ok(_) | Err(_) => OperationResult::Retry("Retrying..."), + } + }); + + let response = match hub_urls { + Ok(resp) => resp, + Err(_) => return Ok(fallback), + }; + + let config: ZosConfig = match response.json() { + Ok(config) => config, + Err(_) => return Ok(fallback), + }; + + if config.hub_url.is_empty() { + Ok(fallback) + } else { + Ok(config.hub_url) + } +} pub struct Repo { name: String, + hub: Vec, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -38,17 +81,41 @@ pub struct Flist { } impl Repo { - pub fn new(name: T) -> Repo + pub fn new(name: T) -> Result where T: AsRef, { - Repo { + let config = crate::config::Config::current()?; + let hub = get_hub_url(&config.runmode)?; + Ok(Repo { name: String::from(name.as_ref()), + hub, + }) + } + + /// Helper function to find the first working hub URL + fn get_working_hub(&self) -> &String { + for hub_url in &self.hub { + if self.is_hub_working(hub_url) { + return hub_url; + } + } + + &self.hub[0] + } + + /// Check if a specific hub URL is working + fn is_hub_working(&self, hub_url: &str) -> bool { + let health_url = format!("{}/api/flist", hub_url); + + match get(&health_url) { + Ok(response) => response.status().is_success(), + Err(_) => false, } } pub fn list(&self) -> Result> { - let url = format!("{}/api/flist/{}", HUB, self.name,); + let url = format!("{}/api/flist/{}", self.get_working_hub(), self.name,); let response = get(&url)?; let mut info: Vec = match response.status() { @@ -56,7 +123,7 @@ impl Repo { s => bail!("failed to get flist info: {}", s), }; for flist in info.iter_mut() { - flist.url = format!("{}/{}/{}", HUB, self.name, flist.name); + flist.url = format!("{}/{}/{}", self.get_working_hub(), self.name, flist.name); } Ok(info) @@ -65,7 +132,12 @@ impl Repo { pub fn list_tag>(&self, tag: S) -> Result>> { let tag = tag.as_ref(); - let url = format!("{}/api/flist/{}/tags/{}", HUB, self.name, tag); + let url = format!( + "{}/api/flist/{}/tags/{}", + self.get_working_hub(), + self.name, + tag + ); let response = get(&url)?; let mut info: Vec = match response.status() { StatusCode::OK => response.json()?, @@ -90,7 +162,7 @@ impl Repo { Some(target) => target, }; - flist.url = format!("{}/{}", HUB, target); + flist.url = format!("{}/{}", self.get_working_hub(), target); } Ok(Some(info)) @@ -101,14 +173,24 @@ impl Repo { where T: AsRef, { - let url = format!("{}/api/flist/{}/{}/light", HUB, self.name, flist.as_ref()); + let url = format!( + "{}/api/flist/{}/{}/light", + self.get_working_hub(), + self.name, + flist.as_ref() + ); let response = get(&url)?; let mut info: Flist = match response.status() { StatusCode::OK => response.json()?, s => bail!("failed to get flist info: {}", s), }; - info.url = format!("{}/{}/{}", HUB, self.name, flist.as_ref()); + info.url = format!( + "{}/{}/{}", + self.get_working_hub(), + self.name, + flist.as_ref() + ); Ok(info) } } @@ -168,18 +250,18 @@ mod tests { use super::*; #[test] fn test_get_flist() -> Result<()> { - let repo = Repo::new(String::from("azmy")); + let repo = Repo::new(String::from("azmy"))?; let flist = repo.get("test.flist")?; assert_eq!(flist.name, "test.flist"); assert_eq!(flist.kind, Kind::Regular); - assert_eq!(flist.url, "https://hub.grid.tf/azmy/test.flist"); + assert!(flist.url.ends_with("/azmy/test.flist")); Ok(()) } #[test] fn test_list_tag() -> Result<()> { - let repo = Repo::new(String::from("tf-autobuilder")); + let repo = Repo::new(String::from("tf-autobuilder"))?; let list = repo.list_tag("3b51aa5")?; @@ -195,7 +277,7 @@ mod tests { #[test] fn test_download_flist() -> Result<()> { - let repo = Repo::new(String::from("azmy")); + let repo = Repo::new(String::from("azmy"))?; let flist = repo.get("test.flist")?; let temp = "/tmp/hub-download-test.flist"; flist.download(temp)?; @@ -214,7 +296,7 @@ mod tests { #[test] fn test_list_repo() -> Result<()> { - let repo = Repo::new(String::from("azmy")); + let repo = Repo::new(String::from("azmy"))?; let lists = repo.list()?; let mut found: Option<&Flist> = None; @@ -227,7 +309,7 @@ mod tests { let found = found.unwrap(); assert_eq!(found.name, "test.flist"); - assert_eq!(found.url, "https://hub.grid.tf/azmy/test.flist"); + assert!(found.url.ends_with("/azmy/test.flist")); Ok(()) } diff --git a/cmds/modules/provisiond/swagger/zos-api.yml b/cmds/modules/provisiond/swagger/zos-api.yml index 1a6440b3..d98e0611 100644 --- a/cmds/modules/provisiond/swagger/zos-api.yml +++ b/cmds/modules/provisiond/swagger/zos-api.yml @@ -375,9 +375,9 @@ components: type: object properties: flist: - type: string + type: array hub_url: - type: string + type: array env: type: object additionalProperties: