diff --git a/Cargo.lock b/Cargo.lock index a8b750b6506..0259d38c45c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -557,12 +557,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base16ct" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" - [[package]] name = "base64" version = "0.21.7" @@ -1144,16 +1138,6 @@ dependencies = [ "toml 0.8.23", ] -[[package]] -name = "cargo_toml" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374b7c592d9c00c1f4972ea58390ac6b18cbb6ab79011f3bdc90a0b82ca06b77" -dependencies = [ - "serde", - "toml 0.9.12+spec-1.1.0", -] - [[package]] name = "cassowary" version = "0.3.0" @@ -2138,19 +2122,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "crucible-client-types" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=7103cd3a3d7b0112d2949dd135db06fef0c156bb#7103cd3a3d7b0112d2949dd135db06fef0c156bb" -dependencies = [ - "base64 0.22.1", - "crucible-workspace-hack", - "schemars 0.8.22", - "serde", - "serde_json", - "uuid", -] - [[package]] name = "crucible-client-types" version = "0.1.0" @@ -3367,7 +3338,7 @@ version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct 0.2.0", + "base16ct", "crypto-bigint", "digest", "ff", @@ -5977,48 +5948,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b024e211b1b371da58cd69e4fb8fa4ed16915edcc0e2e1fb04ac4bad61959f25" -[[package]] -name = "libfalcon" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/falcon?branch=main#d83f56a657e253e09dd16c1c1c62b6eba24d8ad6" -dependencies = [ - "anstyle", - "anyhow", - "base16ct 1.0.0", - "camino", - "cargo_toml 0.22.3", - "clap", - "colored 3.1.1", - "futures", - "indicatif", - "libc", - "libnet", - "oxnet", - "prettyplease", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=36f20be9bb4c3b362029237f5feb6377c982395f)", - "quote", - "rand 0.8.5", - "regex", - "reqwest 0.12.28", - "ron 0.12.0", - "serde", - "sha2", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "smf 0.2.3", - "syn 2.0.117", - "tabwriter", - "thiserror 1.0.69", - "tokio", - "tokio-tungstenite 0.21.0", - "toml 0.9.12+spec-1.1.0", - "uuid", - "xz2", - "zone", -] - [[package]] name = "libgit2-sys" version = "0.18.3+1.9.2" @@ -6261,14 +6190,15 @@ dependencies = [ [[package]] name = "lldpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/lldp#d22509dfdb051321b859e924948605115691b93c" +source = "git+https://github.com/oxidecomputer/lldp?rev=c3305fd1a7ea7aba31f3834757a6b931e4f59fe6#c3305fd1a7ea7aba31f3834757a6b931e4f59fe6" dependencies = [ "chrono", "futures", "lldpd-common", - "progenitor 0.11.2", - "protocol", - "reqwest 0.12.28", + "lldpd-types-versions", + "progenitor 0.13.0", + "protocol 0.1.0 (git+https://github.com/oxidecomputer/lldp?rev=c3305fd1a7ea7aba31f3834757a6b931e4f59fe6)", + "reqwest 0.13.2", "schemars 0.8.22", "serde", "serde_json", @@ -6279,7 +6209,7 @@ dependencies = [ [[package]] name = "lldpd-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/lldp#d22509dfdb051321b859e924948605115691b93c" +source = "git+https://github.com/oxidecomputer/lldp?rev=c3305fd1a7ea7aba31f3834757a6b931e4f59fe6#c3305fd1a7ea7aba31f3834757a6b931e4f59fe6" dependencies = [ "anyhow", "dpd-client 0.1.0 (git+https://github.com/oxidecomputer/dendrite?branch=main)", @@ -6293,6 +6223,18 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "lldpd-types-versions" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/lldp?rev=c3305fd1a7ea7aba31f3834757a6b931e4f59fe6#c3305fd1a7ea7aba31f3834757a6b931e4f59fe6" +dependencies = [ + "chrono", + "protocol 0.1.0 (git+https://github.com/oxidecomputer/lldp?rev=c3305fd1a7ea7aba31f3834757a6b931e4f59fe6)", + "schemars 0.8.22", + "serde", + "uuid", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -6368,17 +6310,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "lzss" version = "0.8.2" @@ -8294,7 +8225,7 @@ dependencies = [ "progenitor-client 0.13.0", "progenitor-extras", "proptest", - "protocol", + "protocol 0.1.0 (git+https://github.com/oxidecomputer/lldp)", "rand 0.9.2", "regress", "reqwest 0.13.2", @@ -8530,6 +8461,7 @@ dependencies = [ "newtype_derive", "omicron-test-utils", "omicron-workspace-hack", + "omicron-zone-package", "parse-display", "petgraph 0.8.3", "serde", @@ -8659,7 +8591,7 @@ dependencies = [ "pretty_assertions", "progenitor-client 0.13.0", "progenitor-extras", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", + "propolis-client", "qorb", "rand 0.9.2", "range-requests", @@ -9104,9 +9036,9 @@ dependencies = [ "oxnet", "pretty_assertions", "progenitor 0.13.0", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", + "propolis-client", "propolis-mock-server", - "propolis_api_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", + "propolis_api_types", "rand 0.9.2", "range-requests", "rcgen", @@ -9188,7 +9120,6 @@ dependencies = [ "hex", "http", "libc", - "libfalcon", "nexus-config", "omicron-common", "omicron-workspace-hack", @@ -9234,7 +9165,7 @@ dependencies = [ "aho-corasick", "anyhow", "aws-lc-rs", - "base16ct 0.2.0", + "base16ct", "base64 0.22.1", "base64ct", "bitflags 1.3.2", @@ -9295,7 +9226,6 @@ dependencies = [ "iddqd", "idna", "indexmap 2.13.0", - "indicatif", "inout", "ipnet", "ipnetwork", @@ -9396,7 +9326,6 @@ dependencies = [ "zeroize", "zip 0.6.6", "zip 4.6.1", - "zone", ] [[package]] @@ -10162,7 +10091,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" dependencies = [ - "base16ct 0.2.0", + "base16ct", "ecdsa", "elliptic-curve", "primeorder", @@ -11214,8 +11143,8 @@ name = "propolis-api-types-versions" version = "0.0.0" source = "git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2#368a2225b79328514ce0ea9181d8f874019edaa2" dependencies = [ - "crucible-client-types 0.1.0 (git+https://github.com/oxidecomputer/crucible?rev=a945a32ba9e1f2098ce3a8963765f1894f37110b)", - "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", + "crucible-client-types", + "propolis_types", "schemars 0.8.22", "serde", "thiserror 1.0.69", @@ -11229,7 +11158,7 @@ source = "git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0 dependencies = [ "async-trait", "base64 0.21.7", - "crucible-client-types 0.1.0 (git+https://github.com/oxidecomputer/crucible?rev=a945a32ba9e1f2098ce3a8963765f1894f37110b)", + "crucible-client-types", "futures", "progenitor 0.13.0", "progenitor-client 0.13.0", @@ -11246,30 +11175,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "propolis-client" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=36f20be9bb4c3b362029237f5feb6377c982395f#36f20be9bb4c3b362029237f5feb6377c982395f" -dependencies = [ - "async-trait", - "base64 0.21.7", - "crucible-client-types 0.1.0 (git+https://github.com/oxidecomputer/crucible?rev=7103cd3a3d7b0112d2949dd135db06fef0c156bb)", - "futures", - "progenitor 0.10.0", - "progenitor-client 0.10.0", - "propolis_api_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=36f20be9bb4c3b362029237f5feb6377c982395f)", - "rand 0.9.2", - "reqwest 0.12.28", - "schemars 0.8.22", - "serde", - "serde_json", - "slog", - "thiserror 1.0.69", - "tokio", - "tokio-tungstenite 0.21.0", - "uuid", -] - [[package]] name = "propolis-mock-server" version = "0.0.0" @@ -11284,8 +11189,8 @@ dependencies = [ "hyper", "progenitor 0.13.0", "propolis-api-types-versions", - "propolis_api_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", - "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", + "propolis_api_types", + "propolis_types", "rand 0.9.2", "reqwest 0.13.2", "schemars 0.8.22", @@ -11308,23 +11213,10 @@ name = "propolis_api_types" version = "0.0.0" source = "git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2#368a2225b79328514ce0ea9181d8f874019edaa2" dependencies = [ - "crucible-client-types 0.1.0 (git+https://github.com/oxidecomputer/crucible?rev=a945a32ba9e1f2098ce3a8963765f1894f37110b)", + "crucible-client-types", "propolis-api-types-versions", ] -[[package]] -name = "propolis_api_types" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=36f20be9bb4c3b362029237f5feb6377c982395f#36f20be9bb4c3b362029237f5feb6377c982395f" -dependencies = [ - "crucible-client-types 0.1.0 (git+https://github.com/oxidecomputer/crucible?rev=7103cd3a3d7b0112d2949dd135db06fef0c156bb)", - "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=36f20be9bb4c3b362029237f5feb6377c982395f)", - "schemars 0.8.22", - "serde", - "thiserror 1.0.69", - "uuid", -] - [[package]] name = "propolis_types" version = "0.0.0" @@ -11334,15 +11226,6 @@ dependencies = [ "serde", ] -[[package]] -name = "propolis_types" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=36f20be9bb4c3b362029237f5feb6377c982395f#36f20be9bb4c3b362029237f5feb6377c982395f" -dependencies = [ - "schemars 0.8.22", - "serde", -] - [[package]] name = "proptest" version = "1.10.0" @@ -11362,6 +11245,17 @@ dependencies = [ "unarray", ] +[[package]] +name = "protocol" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/lldp?rev=c3305fd1a7ea7aba31f3834757a6b931e4f59fe6#c3305fd1a7ea7aba31f3834757a6b931e4f59fe6" +dependencies = [ + "anyhow", + "schemars 0.8.22", + "serde", + "thiserror 1.0.69", +] + [[package]] name = "protocol" version = "0.1.0" @@ -12072,6 +11966,7 @@ dependencies = [ "bytes", "cookie", "cookie_store", + "encoding_rs", "futures-channel", "futures-core", "futures-util", @@ -12084,6 +11979,7 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", "quinn", @@ -12149,20 +12045,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "ron" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" -dependencies = [ - "bitflags 2.11.0", - "once_cell", - "serde", - "serde_derive", - "typeid", - "unicode-ident", -] - [[package]] name = "rpassword" version = "7.4.0" @@ -12870,7 +12752,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct 0.2.0", + "base16ct", "der", "generic-array", "pkcs8", @@ -13394,7 +13276,7 @@ dependencies = [ "omicron-workspace-hack", "oxnet", "progenitor 0.13.0", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", + "propolis-client", "regress", "reqwest 0.13.2", "schemars 0.8.22", @@ -13612,7 +13494,7 @@ dependencies = [ "omicron-workspace-hack", "oxnet", "propolis-api-types-versions", - "propolis_api_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=368a2225b79328514ce0ea9181d8f874019edaa2)", + "propolis_api_types", "proptest", "schemars 0.8.22", "serde", @@ -14922,7 +14804,7 @@ name = "tlvc-text" version = "0.3.0" source = "git+https://github.com/oxidecomputer/tlvc#e644a21a7ca973ed31499106ea926bd63ebccc6f" dependencies = [ - "ron 0.8.1", + "ron", "serde", "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc)", "zerocopy 0.6.6", @@ -17513,7 +17395,7 @@ dependencies = [ "camino", "camino-tempfile", "cargo_metadata 0.21.0", - "cargo_toml 0.21.0", + "cargo_toml", "clap", "dev-tools-common", "fs-err 3.3.0", @@ -17547,15 +17429,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index ebece628a7c..14f9f846899 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -582,11 +582,10 @@ kstat-rs = "0.2.4" libc = "0.2.174" linkme = "0.3" libipcc = { git = "https://github.com/oxidecomputer/ipcc-rs", rev = "524eb8f125003dff50b9703900c6b323f00f9e1b" } -libfalcon = { git = "https://github.com/oxidecomputer/falcon", branch = "main" } libnvme = { git = "https://github.com/oxidecomputer/libnvme", rev = "dd5bb221d327a1bc9287961718c3c10d6bd37da0" } linear-map = "1.2.0" live-tests-macros = { path = "live-tests/macros" } -lldpd_client = { git = "https://github.com/oxidecomputer/lldp", package = "lldpd-client" } +lldpd_client = { git = "https://github.com/oxidecomputer/lldp", rev = "c3305fd1a7ea7aba31f3834757a6b931e4f59fe6", package = "lldpd-client" } lldp_protocol = { git = "https://github.com/oxidecomputer/lldp", package = "protocol" } macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" @@ -1066,9 +1065,6 @@ opt-level = 3 # [patch."https://github.com/oxidecomputer/diesel-dtrace"] # diesel-dtrace = { path = "../diesel-dtrace" } -# [patch."https://github.com/oxidecomputer/falcon"] -# libfalcon = { path = "../falcon/lib" } - # [patch."https://github.com/oxidecomputer/propolis"] # propolis-client = { path = "../propolis/lib/propolis-client" } # propolis-mock-server = { path = "../propolis/bin/mock-server" } diff --git a/dev-tools/ls-apis/Cargo.toml b/dev-tools/ls-apis/Cargo.toml index b1942608144..a06976570a7 100644 --- a/dev-tools/ls-apis/Cargo.toml +++ b/dev-tools/ls-apis/Cargo.toml @@ -16,6 +16,7 @@ iddqd.workspace = true indent_write.workspace = true itertools.workspace = true newtype_derive.workspace = true +omicron-zone-package.workspace = true parse-display.workspace = true petgraph.workspace = true serde.workspace = true diff --git a/dev-tools/ls-apis/src/bin/ls-apis.rs b/dev-tools/ls-apis/src/bin/ls-apis.rs index 7768c988e16..6cfef8da186 100644 --- a/dev-tools/ls-apis/src/bin/ls-apis.rs +++ b/dev-tools/ls-apis/src/bin/ls-apis.rs @@ -299,11 +299,18 @@ impl TryFrom<&LsApis> for LoadArgs { let self_manifest_dir_str = std::env::var("CARGO_MANIFEST_DIR") .context("expected CARGO_MANIFEST_DIR in environment")?; let self_manifest_dir = Utf8PathBuf::from(self_manifest_dir_str); + // The API manifest is at the root of this particular package. let api_manifest_path = args .api_manifest .clone() .unwrap_or_else(|| self_manifest_dir.join("api-manifest.toml")); - Ok(LoadArgs { api_manifest_path }) + + // This package is two levels down from the workspace root. + let mut workspace_root = self_manifest_dir; + workspace_root.pop(); + workspace_root.pop(); + + Ok(LoadArgs { workspace_root, api_manifest_path }) } } diff --git a/dev-tools/ls-apis/src/lib.rs b/dev-tools/ls-apis/src/lib.rs index 62449be95c9..c72c4f4b2ce 100644 --- a/dev-tools/ls-apis/src/lib.rs +++ b/dev-tools/ls-apis/src/lib.rs @@ -82,6 +82,9 @@ impl Borrow for ServerPackageName { /// Parameters for loading information about system APIs pub struct LoadArgs { + /// path to the root of the Omicron workspace + pub workspace_root: Utf8PathBuf, + /// path to developer-maintained API metadata pub api_manifest_path: Utf8PathBuf, } diff --git a/dev-tools/ls-apis/src/system_apis.rs b/dev-tools/ls-apis/src/system_apis.rs index dcc43c4b80c..45a05cdc419 100644 --- a/dev-tools/ls-apis/src/system_apis.rs +++ b/dev-tools/ls-apis/src/system_apis.rs @@ -127,7 +127,8 @@ impl SystemApis { let api_metadata: AllApiMetadata = parse_toml_file(&args.api_manifest_path)?; // Load Cargo metadata and validate it against the manifest. - let (workspaces, warnings) = Workspaces::load(&api_metadata)?; + let (workspaces, warnings) = + Workspaces::load(&api_metadata, &args.workspace_root)?; if !warnings.is_empty() { // We treat these warnings as fatal here. for e in warnings { diff --git a/dev-tools/ls-apis/src/workspaces.rs b/dev-tools/ls-apis/src/workspaces.rs index f3cb89b19d9..674a28e01aa 100644 --- a/dev-tools/ls-apis/src/workspaces.rs +++ b/dev-tools/ls-apis/src/workspaces.rs @@ -7,14 +7,59 @@ use crate::ClientPackageName; use crate::api_metadata::AllApiMetadata; use crate::cargo::Workspace; -use anyhow::{Context, Result, anyhow, ensure}; +use anyhow::{Context, Result, anyhow, bail, ensure}; use camino::Utf8Path; use cargo_metadata::CargoOpt; use cargo_metadata::Package; use cargo_metadata::PackageId; +use omicron_zone_package::package::PackageSource; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::sync::Arc; +use std::sync::LazyLock; + +struct RelatedRepoConfig { + repo_name: &'static str, + expected_pkg_name: &'static str, + extra_cargo_features: Option, +} + +static RELATED_REPOS: LazyLock<[RelatedRepoConfig; 5]> = LazyLock::new(|| { + [ + RelatedRepoConfig { + repo_name: "crucible", + expected_pkg_name: "crucible-agent-client", + extra_cargo_features: None, + }, + RelatedRepoConfig { + repo_name: "dendrite", + expected_pkg_name: "dpd-client", + extra_cargo_features: None, + }, + RelatedRepoConfig { + repo_name: "propolis", + expected_pkg_name: "propolis-client", + // The artifacts shipped from the Propolis repo (particularly, + // `propolis-server`) are built with the `omicron-build` + // feature, which is not enabled by default. Enable this + // feature when loading the Propolis repo metadata so that we + // see the dependency tree that a shipping system will have. + extra_cargo_features: Some(CargoOpt::SomeFeatures(vec![ + String::from("omicron-build"), + ])), + }, + RelatedRepoConfig { + repo_name: "maghemite", + expected_pkg_name: "mg-admin-client", + extra_cargo_features: None, + }, + RelatedRepoConfig { + repo_name: "lldp", + expected_pkg_name: "lldpd-client", + extra_cargo_features: None, + }, + ] +}); /// Thin wrapper around a list of workspaces that makes it easy to query which /// workspace has which package @@ -32,6 +77,7 @@ impl Workspaces { /// of inconsistencies between API metadata and Cargo metadata. pub fn load( api_metadata: &AllApiMetadata, + workspace_root: &Utf8Path, ) -> Result<(Workspaces, Vec)> { // First, load information about the "omicron" workspace. This is the // current workspace so we don't need to provide the path to it. @@ -43,6 +89,23 @@ impl Workspaces { ignored_non_clients, )?); + // Next, load the top level package manifest. This will tell us for + // each related component (like Crucible, Maghemite, etc.), which commit + // of that component's repo Omicron actually deploys to running systems. + let package_manifest_path = + workspace_root.join("package-manifest.toml"); + let package_manifest = + omicron_zone_package::config::parse(&package_manifest_path) + .with_context(|| { + format!("parsing {package_manifest_path:?}") + })?; + let mut related_repo_commits = BTreeMap::new(); + for related_repo in &*RELATED_REPOS { + let commit = + find_repo_commit(&package_manifest, related_repo.repo_name)?; + related_repo_commits.insert(related_repo.repo_name, commit); + } + // In order to assemble this metadata, Cargo already has a clone of most // of the other workspaces that we care about. We'll use those clones // rather than manage our own. @@ -58,43 +121,31 @@ impl Workspaces { // // If we had many more repos than this, we'd probably want to limit the // concurrency. - let handles: Vec<_> = [ - // To find this repo ... look up this package in Omicron - // | | +---- and enable these extra - // | | | features when loading - // v v v - ("crucible", "crucible-agent-client", None), - ("dendrite", "dpd-client", None), - ( - "propolis", - "propolis-client", - // The artifacts shipped from the Propolis repo (particularly, - // `propolis-server`) are built with the `omicron-build` - // feature, which is not enabled by default. Enable this - // feature when loading the Propolis repo metadata so that we - // see the dependency tree that a shipping system will have. - Some(CargoOpt::SomeFeatures(vec![String::from( - "omicron-build", - )])), - ), - ("maghemite", "mg-admin-client", None), - ("lldp", "lldpd-client", None), - ] - .into_iter() - .map(|(repo, omicron_pkg, extra_features)| { - let mine = omicron.clone(); - let my_ignored = ignored_non_clients.clone(); - std::thread::spawn(move || { - load_dependent_repo( - &mine, - repo, - omicron_pkg, - extra_features, - my_ignored, - ) + let handles: Vec<_> = RELATED_REPOS + .iter() + .map(|repo_config| { + let RelatedRepoConfig { + repo_name, + expected_pkg_name, + extra_cargo_features, + } = repo_config; + let mine = omicron.clone(); + let my_ignored = ignored_non_clients.clone(); + // unwrap(): we loaded a commit for each repo in the loop above + let expected_commit = + (*related_repo_commits.get(repo_name).unwrap()).clone(); + std::thread::spawn(move || { + load_dependent_repo( + &mine, + repo_name, + expected_pkg_name, + extra_cargo_features.clone(), + my_ignored, + &expected_commit, + ) + }) }) - }) - .collect(); + .collect(); let mut workspaces: BTreeMap<_, _> = handles .into_iter() @@ -111,23 +162,6 @@ impl Workspaces { Arc::into_inner(omicron).expect("no more Omicron Arc references"), ); - // To load Dendrite, we need to look something up in Maghemite (loaded - // above). - let maghemite = workspaces - .get("maghemite") - .ok_or_else(|| anyhow!("missing maghemite workspaces"))?; - - workspaces.insert( - String::from("dendrite"), - load_dependent_repo( - &maghemite, - "dendrite", - "dpd-client", - None, - ignored_non_clients.clone(), - )?, - ); - // Validate the metadata against what we found in the workspaces. let mut client_pkgnames_unused: BTreeSet<_> = api_metadata.client_pkgnames().collect(); @@ -221,26 +255,61 @@ fn load_dependent_repo( pkgname: &str, extra_features: Option, ignored_non_clients: BTreeSet, + expected_commit: &str, ) -> Result { - // `Workspace` doesn't let us look up a non-workspace package by name - // because there may be many of them. So list all the pkgids and take any - // one of them -- any of them should work for our purpoes. - let pkgid = workspace.pkgids(pkgname).next().ok_or_else(|| { - anyhow!( - "workspace {} did not contain expected package {}", - workspace.name(), - pkgname - ) - })?; + // It's possible to have more than one non-workspace package with a given + // name. For example, Omicron references `dpd-client` in multiple ways: + // from Nexus and through lldpd-client. So which version do we want? Well, + // the goal of looking up `dpd-client` is really to find the source for + // Dendrite. And we want the Dendrite that's actually going to run on a + // real system. That's the one specified by package-manifest.toml. The + // caller already figured out which commit that is and provided it in + // `expected_commit`. + // + // In summary: we're going to choose the package matching `expected_commit`. + let mut found_pkg = None; + + for pkgid in workspace.pkgids(pkgname) { + let pkginfo = workspace.pkg_by_id(pkgid).with_context(|| { + format!("failed to find metadata for found package {pkgid:?}") + })?; + + let Some(source) = &pkginfo.source else { + eprintln!( + "warn: looking up {pkgid:?}: unexpectedly found source `None`" + ); + continue; + }; + + // This is cheesy, but it works okay for now and fails safely. + if source.repr.contains(expected_commit) { + found_pkg = Some(pkginfo); + break; + } - // Now we can look up the package metadata. - let pkg = workspace.pkg_by_id(pkgid).ok_or_else(|| { - anyhow!( - "workspace {}: did not contain expected package id {}", - workspace.name(), - pkgname - ) - })?; + eprintln!( + "warn: looking up {pkgid:?}: looking for git commit \ + {expected_commit} (based on package-manifest.toml), found \ + source {source:?}" + ); + eprintln!( + "If another version of package {pkgname:?} is found corresponding \ + with this commit, then it may be suspicious to have multiple version \ + of this package, but it will not break this tool." + ); + eprintln!( + "If not, there's a mismatch between commits in package-manifest.toml \ + and Cargo.toml or there is a bug in this tool." + ); + } + + let Some(pkg) = found_pkg else { + bail!( + "found no versions of package {pkgname:?} matching the git commit \ + found in package-manifest.toml ({})", + expected_commit, + ); + }; // The package metadata should show where the package's manifest file should // be. This may be buried deep in the workspace. How do we find the root @@ -284,3 +353,41 @@ fn load_dependent_repo( &ignored_non_clients, ) } + +fn find_repo_commit( + package_manifest: &omicron_zone_package::config::Config, + repo_name: &str, +) -> Result { + // We want to figure out which version of repo `repo_name` is actually + // deployed on real systems. To do that, we look at what's in the package + // manifest. All of the repos we care about are packaged into Omicron as + // prebuilt packages. The one hitch is that the same repo may appear + // multiple times in the package manifest. In that case, we require that + // they all be the same commit. + let candidates: BTreeSet<&String> = package_manifest + .packages + .iter() + .filter_map(|(_, pkg)| match &pkg.source { + PackageSource::Local { .. } + | PackageSource::Composite { .. } + | PackageSource::Manual => None, + PackageSource::Prebuilt { repo, commit, .. } => { + if repo == repo_name { Some(commit) } else { None } + } + }) + .collect(); + + if candidates.is_empty() { + bail!("expected repo {repo_name:?} to appear in package-manifest.toml"); + } + + if candidates.len() > 1 { + bail!( + "expected repo {repo_name:?} to have exactly one git commit \ + in package-manifest.toml, but found multiple: {candidates:?}", + ); + } + + // unwrap(): we already checked that there's at least one + Ok(candidates.into_iter().next().unwrap().clone()) +} diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index d5ea2f8154c..b7bd5b3faaa 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -87,7 +87,7 @@ Oximeter (client: oximeter-client) consumed by: omicron-nexus (omicron/nexus) via 2 paths Propolis (client: propolis-client) - consumed by: omicron-nexus (omicron/nexus) via 3 paths + consumed by: omicron-nexus (omicron/nexus) via 2 paths consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths Crucible Repair (client: repair-client) diff --git a/package-manifest.toml b/package-manifest.toml index 7b2282cc31b..fba7fe0b907 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -720,8 +720,8 @@ output.intermediate_only = true service_name = "lldp" source.type = "prebuilt" source.repo = "lldp" -source.commit = "61479b6922f9112fbe1e722414d2b8055212cb12" -source.sha256 = "8f988c0b0fa3ad4121ab0e825298601035e56c5c054bdc3a1dfb4d6c8fd5b300" +source.commit = "c3305fd1a7ea7aba31f3834757a6b931e4f59fe6" +source.sha256 = "56603114fb9bce8855e80c6a8d4583eaa8fafe3edb03a5268950453dbf5c08e6" output.type = "zone" output.intermediate_only = true diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index af83e43beea..ad4b8d303c7 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -50,12 +50,3 @@ serde.workspace = true [features] seed-gen = ["dep:filetime"] - -[target.'cfg(target_os = "illumos")'.dependencies] -libfalcon.workspace = true - -[[bin]] -name = "falcon_runner" - -[[bin]] -name = "falcon_runner_cli" diff --git a/test-utils/README.md b/test-utils/README.md deleted file mode 100644 index d6ae4759c61..00000000000 --- a/test-utils/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# Falcon VM based testing - -It is now possible to run omicron tests in a VM. Any test can be run in a VM, -but many of them will not work, because their dependencies are not installed. -This is possible, and can be done in a number of ways. We just need to think -about how we want to do it. - -Running an individual test in a VM currently works by: - - 1. Building the test binary - 2. Copying it into a temporary directory - 3. Mounting that temporary directory inside the VM - 4. Logging into and running the binary with the given test name over the - serial port. - -## Running an Omicron test - -An individual test can be run in the following manner: - -```shell -cd omicron -CARGO_TARGET_X86_64_UNKNOWN_ILLUMOS_RUNNER=./test-utils/src/bin/falcon_runner.sh cargo nextest run -p --nocapture -``` - -A concrete example, which can be used for sanity checking that the test actually runs -in a VM, is provided in the `omicron-test-utils` package, and named `launch`. - -```shell -CARGO_TARGET_X86_64_UNKNOWN_ILLUMOS_RUNNER=./test-utils/src/bin/falcon_runner.sh cargo nextest run -p omicron-test-utils launch --nocapture -``` - -## Logging into a test VM - -On failure, the test VM remains running for debugging purposes. All VMs -currently share the name `launchpad_mcduck_test_vm`, although the falcon -deployment metadata resides in a unique temp directory. At the start of each -test, and upon failure this directory is printed to the console. - -``` -Setting falcon directory to /tmp/.tmpXcLx5g -... -Test failed: VM remains running, with falcon dir: /tmp/.tmpXcLx5g -``` - -A user can login to that VM via the serial port with the following command: - -```shell -cargo run -p omicron-test-utils --bin falcon_runner_cli serial launchpad_mcduck_test_vm -f /tmp/.tmpXcLx5g -``` - -Similarly, a user can execute a command on the VM via: - -```shell -cargo run -p omicron-test-utils --bin falcon_runner_cli exec launchpad_mcduck_test_vm '' -f /tmp/.tmpXcLx5g -``` - -This can be used to cat log files, etc.. - -Once a user is done debugging the VM, they should tear it and its associated -resources down with the following command. Note that here, you don't have to -name the VM, as the entire deployment is torn down. However, you do need to -use `pfexec`. - -```shell -pfexec cargo run -p omicron-test-utils --bin falcon_runner_cli destroy -f /tmp/.tmpXcLx5g -``` - -## Troubleshooting - -If for some reason you fail to tear down your VM and need to do it manually you -must remove the VM and its backing filesystems. Don't worry about the falcon -dirs, as they are just small directories in `/tmp`. - -VM backing images are created in zfs under `rpool/falcon/topo`. These can be -destroyed via the following command. Note that this will destroy the backing -filesystems for **all** of your falcon VMs. If you want to be more precise, you -should `zfs list` and figure out what you want to kill. - -```shell -pfexec zfs destroy -r rpool/falcon/topo -``` - -Similarly, all your bhyve backed VMs live in `/dev/vmm`. All VMs can be torn -down with the following command, although you should be careful when doing this -on shared infrastructure. - -```shell -for i in `ls /dev/vmm`; do pfexec bhyvectl --vm=$i --destroy; done -``` - -For more troubleshooting info, see the [falcon wiki](https://github.com/oxidecomputer/falcon/wiki/Reference) - -## Current Limitations - -* Tests must be self-contained and not rely on dependencies. E2E tests will -not currently work. We can easily install pre-reqs via our scripts, but this -will delay each test run. We may instead want to build falcon base images with -the pre-reqs included. There's a tradeoff here of test run time, vs getting the -latest bits. -* The names of the test runner, and VM are currently hardcoded, and start with -`launchpad_mcduck`. Therefore, tests can only be run serially for now. This is -an easy limitation to lift, we just need to figure out a naming convention and -do it. -* The commands to run tests are quite verbose. We should really have a wrapper -script. - - - - - diff --git a/test-utils/src/bin/falcon_runner.rs b/test-utils/src/bin/falcon_runner.rs deleted file mode 100644 index 243db68d511..00000000000 --- a/test-utils/src/bin/falcon_runner.rs +++ /dev/null @@ -1,107 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Mechanisms to launch and control propolis VMs via falcon - -#[cfg(target_os = "illumos")] -mod illumos { - pub use anyhow::anyhow; - pub use camino_tempfile::tempdir; - pub use libfalcon::{Runner, unit::gb}; - pub use omicron_test_utils::dev::test_setup_log; - pub use slog::info; - pub use std::env; -} - -fn main() -> Result<(), anyhow::Error> { - oxide_tokio_rt::run(main_impl()) -} - -#[cfg(target_os = "illumos")] -async fn main_impl() -> Result<(), anyhow::Error> { - use illumos::*; - let args: Vec<_> = env::args().collect(); - let logctx = test_setup_log("falcon_test_runner"); - let log = logctx.log; - info!(log, "args = {:?}", args); - let name = "launchpad_mcduck"; - - let runner_name = format!("{}_runner", name); - let node_name = format!("{}_test_vm", name); - let mut d = Runner::new(&runner_name); - let vm = d.node(&node_name, "helios-2.0", 2, gb(8)); - - let falcon_dir = tempdir()?; - info!(log, "Setting falcon directory to {}", falcon_dir.path()); - d.set_falcon_dir(Some(falcon_dir.path().to_string())); - - let cargo_bay = tempdir()?; - info!(log, "Set cargo-bay to {} on host machine", cargo_bay.path()); - let source_test_path = &args[1]; - let test_name = &args[3]; - let test_file_name = camino::Utf8Path::new(source_test_path) - .file_name() - .expect("Failed to get test file name"); - let test_path = cargo_bay.path().to_path_buf().join(test_file_name); - - info!(log, "Copying {source_test_path} to {test_path}"); - std::fs::copy(camino::Utf8Path::new(source_test_path), &test_path)?; - - d.mount(cargo_bay.path(), "/opt/cargo-bay", vm)?; - - info!(log, "Launching test vm {node_name}"); - d.launch().await?; - info!(log, "Launched test vm {node_name}"); - - info!(log, "Running test: {test_file_name}::{test_name}"); - let run_test = format!( - "cd /opt/cargo-bay && chmod +x {test_file_name}; \ - res=$?; \ - if [[ $res -eq 0 ]]; then \ - ./{test_file_name} --color never --exact {test_name} --nocapture; \ - res=$?; \ - fi; \ - echo $res" - ); - let out = d.exec(vm, &run_test).await?; - - // The last line of our output contains the exit code - let exit_code_index = out.rfind('\n').expect("No newline found in output"); - - // Ensure there is enough data left for an exit code - let exit_code: u8 = if exit_code_index + 1 < out.len() { - (&out[exit_code_index + 1..]).parse().expect("Invalid exit code") - } else { - panic!("No exit code available"); - }; - - info!(log, "{}", &out[..=exit_code_index]); - - if exit_code == 0u8 { - // We destroy here so that our tempdir doesn't get dropped first and - // give us an error. - // - // `drop` calls `d.destroy()` and also makes sure it doesn't run twice. - drop(d); - Ok(()) - } else { - // Leave the VM running - d.persistent = true; - - // Don't remove the falcon directory - info!( - log, - "Test failed: VM remains running, with falcon dir: {}", - falcon_dir.path() - ); - std::mem::forget(falcon_dir); - - Err(anyhow!("Test failed: exit code = {exit_code}")) - } -} - -#[cfg(not(target_os = "illumos"))] -async fn main_impl() -> Result<(), anyhow::Error> { - Ok(()) -} diff --git a/test-utils/src/bin/falcon_runner.sh b/test-utils/src/bin/falcon_runner.sh deleted file mode 100755 index 32645a6aea1..00000000000 --- a/test-utils/src/bin/falcon_runner.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Override on the command line for your path to Omicron -OMICRON=${OMICRON:-~/omicron} - -if [[ "$*" == *--list* ]]; then - exec "$@" -else - cargo build -p omicron-test-utils --bin falcon_runner - pfexec $OMICRON/target/debug/falcon_runner $@ -fi diff --git a/test-utils/src/bin/falcon_runner_cli.rs b/test-utils/src/bin/falcon_runner_cli.rs deleted file mode 100644 index 47811185e79..00000000000 --- a/test-utils/src/bin/falcon_runner_cli.rs +++ /dev/null @@ -1,25 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2022 Oxide Computer Company - -#[cfg(target_os = "illumos")] -mod illumos { - pub use libfalcon::{Runner, cli::run, error::Error, unit::gb}; -} - -#[cfg(target_os = "illumos")] -fn main() -> Result<(), illumos::Error> { - use illumos::*; - oxide_tokio_rt::run(async { - let mut d = Runner::new("launchpad_mcduck_runner"); - - d.node("launchpad_mcduck_test_vm", "helios-2.0", 2, gb(2)); - run(&mut d).await?; - Ok(()) - }) -} - -#[cfg(not(target_os = "illumos"))] -fn main() {} diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 6288b30b730..a08144757b4 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -398,7 +398,6 @@ errno = { version = "0.3.14" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.1", default-features = false, features = ["std", "sys_rng"] } hyper-rustls = { version = "0.27.7", features = ["http2", "ring", "webpki-tokio"] } hyper-util = { version = "0.1.20", features = ["full"] } -indicatif = { version = "0.18.4", features = ["rayon"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.1.1", features = ["net", "os-ext"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } @@ -407,7 +406,6 @@ rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.3", features = [" tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_datetime-3b31131e45eafb45 = { package = "toml_datetime", version = "0.6.11", default-features = false, features = ["serde"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } -zone = { version = "0.3.1", features = ["async"] } [target.x86_64-unknown-illumos.build-dependencies] cookie = { version = "0.18.1", default-features = false, features = ["percent-encode"] } @@ -417,7 +415,6 @@ errno = { version = "0.3.14" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.1", default-features = false, features = ["std", "sys_rng"] } hyper-rustls = { version = "0.27.7", features = ["http2", "ring", "webpki-tokio"] } hyper-util = { version = "0.1.20", features = ["full"] } -indicatif = { version = "0.18.4", features = ["rayon"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.1.1", features = ["net", "os-ext"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } @@ -426,6 +423,5 @@ rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.3", features = [" tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_datetime-3b31131e45eafb45 = { package = "toml_datetime", version = "0.6.11", default-features = false, features = ["serde"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } -zone = { version = "0.3.1", features = ["async"] } ### END HAKARI SECTION