From 4d90c24399f21d15c239f77ccd59c232c38f77ef Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 15 May 2026 22:26:45 +0000 Subject: [PATCH 1/3] feat: fall back to OCI registry API when Docker is unavailable in pyhl setup Signed-off-by: danbugs --- host/Cargo.lock | 237 +++++++++++++++++++++++++++++++++++++++++++++++ host/Cargo.toml | 3 + host/src/pyhl.rs | 116 +++++++++++++++++++---- 3 files changed, 338 insertions(+), 18 deletions(-) diff --git a/host/Cargo.lock b/host/Cargo.lock index 541109c..7d8c0c0 100644 --- a/host/Cargo.lock +++ b/host/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -181,6 +187,12 @@ dependencies = [ "syn", ] +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + [[package]] name = "cc" version = "1.2.53" @@ -303,6 +315,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -376,6 +397,26 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.8" @@ -392,6 +433,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -512,6 +563,22 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "hyperlight-common" version = "0.15.0" @@ -574,12 +641,15 @@ dependencies = [ "anyhow", "base64", "clap", + "flate2", "hyperlight-host", "libc", "memmap2", "nix", "serde_json", "socket2", + "tar", + "ureq", "windows-sys 0.61.2", ] @@ -832,6 +902,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.1" @@ -878,6 +954,16 @@ dependencies = [ "rapidhash", ] +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mshv-bindings" version = "0.6.6" @@ -1105,6 +1191,20 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rust-embed" version = "8.11.0" @@ -1150,6 +1250,54 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1265,6 +1413,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "smallvec" version = "1.15.1" @@ -1302,6 +1456,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.114" @@ -1324,6 +1484,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -1425,6 +1596,41 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -1437,6 +1643,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -1601,6 +1813,15 @@ dependencies = [ "semver", ] +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1927,6 +2148,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.1" @@ -1991,6 +2222,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + [[package]] name = "zerotrie" version = "0.2.3" diff --git a/host/Cargo.toml b/host/Cargo.toml index 835e77a..a503168 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -36,6 +36,9 @@ memmap2 = "0.9" serde_json = "1" base64 = "0.22" socket2 = { version = "0.5", features = ["all"] } +ureq = "3" +flate2 = "1" +tar = "0.4" [target.'cfg(unix)'.dependencies] nix = { version = "0.29", features = ["fs"] } diff --git a/host/src/pyhl.rs b/host/src/pyhl.rs index 2335941..4abd302 100644 --- a/host/src/pyhl.rs +++ b/host/src/pyhl.rs @@ -406,25 +406,21 @@ pub fn ghcr_image_ref(repo: &str, tag: Option<&str>) -> String { format!("{}:{}", repo, tag.unwrap_or("latest")) } -/// Pull a single file out of an OCI image hosted on GHCR. Uses whichever -/// of `docker` / `podman` is on `$PATH`. The published images are -/// FROM-scratch with a single payload file at a known path (`/kernel` -/// or `/initrd.cpio`), so this is a straightforward -/// `pull → create → cp → rm` dance. -/// -/// Returns `Err` if neither container tool is available — callers that -/// want a pure-library alternative should use -/// [`InstallSource::LocalDir`] or [`InstallSource::Explicit`] instead. +/// Pull a single file out of an OCI image hosted on GHCR. Tries +/// `docker` / `podman` first; if neither is on `$PATH`, falls back to +/// the OCI registry HTTP API (no container runtime needed). pub fn extract_from_ghcr(image: &str, src_path_in_image: &str, dst: &Path) -> Result<()> { - use std::process::Command; + match find_on_path(&["docker", "podman"]) { + Some(tool) => extract_via_docker(tool, image, src_path_in_image, dst), + None => { + eprintln!("pyhl: docker/podman not found, pulling via OCI registry API"); + extract_via_oci_api(image, src_path_in_image, dst) + } + } +} - let tool = find_on_path(&["docker", "podman"]).ok_or_else(|| { - anyhow!( - "need `docker` or `podman` on $PATH to pull the pyhl image from GHCR.\n\ - alternatives: install one of them, or use `pyhl setup --from ` \ - (library callers: InstallSource::LocalDir / ::Explicit)." - ) - })?; +fn extract_via_docker(tool: &str, image: &str, src_path_in_image: &str, dst: &Path) -> Result<()> { + use std::process::Command; let run = |cmd: &mut Command, label: &str| -> Result { let out = cmd @@ -451,7 +447,6 @@ pub fn extract_from_ghcr(image: &str, src_path_in_image: &str, dst: &Path) -> Re "create", )?; - // Guard so the tmp container is removed even if `cp` fails. let _cleanup = GhcrCleanup { tool: tool.to_string(), cname: cname.clone(), @@ -470,6 +465,91 @@ pub fn extract_from_ghcr(image: &str, src_path_in_image: &str, dst: &Path) -> Re Ok(()) } +/// Pull a single file from a GHCR OCI image using the registry HTTP API. +/// No container runtime required. +fn extract_via_oci_api(image: &str, src_path_in_image: &str, dst: &Path) -> Result<()> { + use flate2::read::GzDecoder; + use tar::Archive; + + let (repo, tag) = parse_image_ref(image)?; + + eprintln!("pyhl: pull {image} (via OCI API)"); + + let token_url = format!( + "https://ghcr.io/token?scope=repository:{repo}:pull" + ); + let token_body: serde_json::Value = serde_json::from_slice( + &ureq::get(&token_url) + .call() + .with_context(|| format!("fetch GHCR auth token for {repo}"))? + .into_body() + .read_to_vec()?, + )?; + let token = token_body["token"] + .as_str() + .ok_or_else(|| anyhow!("GHCR token response missing 'token' field"))?; + + let manifest_url = format!( + "https://ghcr.io/v2/{repo}/manifests/{tag}" + ); + let manifest: serde_json::Value = serde_json::from_slice( + &ureq::get(&manifest_url) + .header("Authorization", &format!("Bearer {token}")) + .header( + "Accept", + "application/vnd.oci.image.manifest.v1+json,\ + application/vnd.docker.distribution.manifest.v2+json", + ) + .call() + .with_context(|| format!("fetch manifest for {repo}:{tag}"))? + .into_body() + .read_to_vec()?, + )?; + + let digest = manifest["layers"] + .as_array() + .and_then(|layers| layers.first()) + .and_then(|l| l["digest"].as_str()) + .ok_or_else(|| anyhow!("manifest has no layers for {repo}:{tag}"))?; + + let blob_url = format!("https://ghcr.io/v2/{repo}/blobs/{digest}"); + let body = ureq::get(&blob_url) + .header("Authorization", &format!("Bearer {token}")) + .call() + .with_context(|| format!("fetch blob {digest}"))? + .into_body() + .read_to_vec()?; + + let target_name = src_path_in_image.trim_start_matches('/'); + let gz = GzDecoder::new(body.as_slice()); + let mut archive = Archive::new(gz); + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?; + if path.to_str().is_some_and(|p| p.trim_start_matches("./") == target_name) { + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + let mut out = std::fs::File::create(dst)?; + std::io::copy(&mut entry, &mut out)?; + return Ok(()); + } + } + + bail!("file '{target_name}' not found in OCI layer for {repo}:{tag}") +} + +/// Parse "ghcr.io/org/repo:tag" into ("org/repo", "tag"). +fn parse_image_ref(image: &str) -> Result<(&str, &str)> { + let without_registry = image + .strip_prefix("ghcr.io/") + .ok_or_else(|| anyhow!("expected ghcr.io/ prefix in image ref: {image}"))?; + match without_registry.rsplit_once(':') { + Some((repo, tag)) => Ok((repo, tag)), + None => Ok((without_registry, "latest")), + } +} + struct GhcrCleanup { tool: String, cname: String, From 2c6331216e330c2ba87dfaf75aa7099d6de9e8ee Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 15 May 2026 22:30:08 +0000 Subject: [PATCH 2/3] fix: raise ureq body limit for large OCI blobs (initrd is 62 MiB) Signed-off-by: danbugs --- host/src/pyhl.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/host/src/pyhl.rs b/host/src/pyhl.rs index 4abd302..6311a36 100644 --- a/host/src/pyhl.rs +++ b/host/src/pyhl.rs @@ -518,6 +518,8 @@ fn extract_via_oci_api(image: &str, src_path_in_image: &str, dst: &Path) -> Resu .call() .with_context(|| format!("fetch blob {digest}"))? .into_body() + .with_config() + .limit(512 * 1024 * 1024) .read_to_vec()?; let target_name = src_path_in_image.trim_start_matches('/'); From ff1c4ab2bfb242e0c912e6a1118eeba9217bc770 Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 15 May 2026 22:32:02 +0000 Subject: [PATCH 3/3] style: cargo fmt Signed-off-by: danbugs --- host/src/pyhl.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/host/src/pyhl.rs b/host/src/pyhl.rs index 6311a36..bc1adf6 100644 --- a/host/src/pyhl.rs +++ b/host/src/pyhl.rs @@ -475,9 +475,7 @@ fn extract_via_oci_api(image: &str, src_path_in_image: &str, dst: &Path) -> Resu eprintln!("pyhl: pull {image} (via OCI API)"); - let token_url = format!( - "https://ghcr.io/token?scope=repository:{repo}:pull" - ); + let token_url = format!("https://ghcr.io/token?scope=repository:{repo}:pull"); let token_body: serde_json::Value = serde_json::from_slice( &ureq::get(&token_url) .call() @@ -489,9 +487,7 @@ fn extract_via_oci_api(image: &str, src_path_in_image: &str, dst: &Path) -> Resu .as_str() .ok_or_else(|| anyhow!("GHCR token response missing 'token' field"))?; - let manifest_url = format!( - "https://ghcr.io/v2/{repo}/manifests/{tag}" - ); + let manifest_url = format!("https://ghcr.io/v2/{repo}/manifests/{tag}"); let manifest: serde_json::Value = serde_json::from_slice( &ureq::get(&manifest_url) .header("Authorization", &format!("Bearer {token}")) @@ -528,7 +524,10 @@ fn extract_via_oci_api(image: &str, src_path_in_image: &str, dst: &Path) -> Resu for entry in archive.entries()? { let mut entry = entry?; let path = entry.path()?; - if path.to_str().is_some_and(|p| p.trim_start_matches("./") == target_name) { + if path + .to_str() + .is_some_and(|p| p.trim_start_matches("./") == target_name) + { if let Some(parent) = dst.parent() { std::fs::create_dir_all(parent)?; }