Skip to content

Commit 3e31eab

Browse files
authored
refactor: extract materialized_artifact crate out of fspy (#344)
1 parent 0e29f32 commit 3e31eab

File tree

18 files changed

+423
-184
lines changed

18 files changed

+423
-184
lines changed

Cargo.lock

Lines changed: 17 additions & 19 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ cc = "1.2.39"
5757
clap = "4.5.53"
5858
color-eyre = "0.6.5"
5959
compact_str = "0.9.0"
60-
const_format = "0.2.34"
6160
constcat = "0.6.1"
6261
copy_dir = "0.1.3"
6362
cow-utils = "0.1.3"
@@ -70,11 +69,13 @@ derive_more = "2.0.1"
7069
diff-struct = "0.5.3"
7170
directories = "6.0.0"
7271
elf = { version = "0.8.0", default-features = false }
72+
materialized_artifact = { path = "crates/materialized_artifact" }
73+
materialized_artifact_build = { path = "crates/materialized_artifact_build" }
7374
flate2 = "1.0.35"
7475
fspy = { path = "crates/fspy" }
7576
fspy_detours_sys = { path = "crates/fspy_detours_sys" }
76-
fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib" }
77-
fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib" }
77+
fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" }
78+
fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib", target = "target" }
7879
fspy_seccomp_unotify = { path = "crates/fspy_seccomp_unotify" }
7980
fspy_shared = { path = "crates/fspy_shared" }
8081
fspy_shared_unix = { path = "crates/fspy_shared_unix" }
@@ -103,7 +104,6 @@ pretty_assertions = "1.4.1"
103104
pty_terminal = { path = "crates/pty_terminal" }
104105
pty_terminal_test = { path = "crates/pty_terminal_test" }
105106
pty_terminal_test_client = { path = "crates/pty_terminal_test_client" }
106-
rand = "0.9.1"
107107
ratatui = "0.30.0"
108108
rayon = "1.10.0"
109109
ref-cast = "1.0.24"

crates/fspy/Cargo.toml

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,36 +9,30 @@ allocator-api2 = { workspace = true, features = ["alloc"] }
99
wincode = { workspace = true }
1010
bstr = { workspace = true, default-features = false }
1111
bumpalo = { workspace = true }
12-
const_format = { workspace = true, features = ["fmt"] }
1312
derive_more = { workspace = true, features = ["debug"] }
13+
materialized_artifact = { workspace = true }
1414
fspy_shared = { workspace = true }
1515
futures-util = { workspace = true }
1616
libc = { workspace = true }
1717
ouroboros = { workspace = true }
18-
rand = { workspace = true }
1918
rustc-hash = { workspace = true }
2019
tempfile = { workspace = true }
2120
thiserror = { workspace = true }
2221
tokio = { workspace = true, features = ["net", "process", "io-util", "sync", "rt"] }
2322
tokio-util = { workspace = true }
2423
which = { workspace = true, features = ["tracing"] }
25-
xxhash-rust = { workspace = true }
2624

2725
[target.'cfg(target_os = "linux")'.dependencies]
2826
fspy_seccomp_unotify = { workspace = true, features = ["supervisor"] }
2927
nix = { workspace = true, features = ["uio"] }
3028
tokio = { workspace = true, features = ["bytes"] }
3129

32-
[target.'cfg(all(unix, not(target_env = "musl")))'.dependencies]
33-
fspy_preload_unix = { workspace = true }
34-
3530
[target.'cfg(unix)'.dependencies]
3631
fspy_shared_unix = { workspace = true }
3732
nix = { workspace = true, features = ["fs", "process", "socket", "feature"] }
3833

3934
[target.'cfg(target_os = "windows")'.dependencies]
4035
fspy_detours_sys = { workspace = true }
41-
fspy_preload_windows = { workspace = true }
4236
winapi = { workspace = true, features = ["winbase", "securitybaseapi", "handleapi"] }
4337
winsafe = { workspace = true }
4438

@@ -59,11 +53,19 @@ fspy_test_bin = { path = "../fspy_test_bin", artifact = "bin", target = "aarch64
5953
[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dev-dependencies]
6054
fspy_test_bin = { path = "../fspy_test_bin", artifact = "bin", target = "x86_64-unknown-linux-musl" }
6155

56+
# Artifact build-deps must be unconditional: cargo's resolver panics when
57+
# `artifact = "cdylib"` deps live under a `[target.cfg.build-dependencies]`
58+
# block on cross-compile. Each preload crate's source is cfg-gated to compile
59+
# as an empty cdylib on non-applicable targets, so the unused cross-target
60+
# builds are cheap.
6261
[build-dependencies]
6362
anyhow = { workspace = true }
63+
materialized_artifact_build = { workspace = true }
6464
flate2 = { workspace = true }
65+
fspy_preload_unix = { workspace = true }
66+
fspy_preload_windows = { workspace = true }
67+
sha2 = { workspace = true }
6568
tar = { workspace = true }
66-
xxhash-rust = { workspace = true, features = ["xxh3"] }
6769

6870
[lints]
6971
workspace = true
@@ -72,4 +74,4 @@ workspace = true
7274
doctest = false
7375

7476
[package.metadata.cargo-shear]
75-
ignored = ["ctor", "fspy_test_bin"]
77+
ignored = ["ctor", "fspy_test_bin", "fspy_preload_unix", "fspy_preload_windows"]

crates/fspy/build.rs

Lines changed: 91 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
use std::{
2-
env::{self, current_dir},
2+
env,
3+
fmt::Write as _,
34
fs,
45
io::{Cursor, Read},
5-
path::Path,
6+
path::{Path, PathBuf},
67
process::{Command, Stdio},
78
};
89

910
use anyhow::{Context, bail};
10-
use xxhash_rust::xxh3::xxh3_128;
11+
use sha2::{Digest, Sha256};
1112

12-
fn download(url: &str) -> anyhow::Result<impl Read + use<>> {
13+
fn download(url: &str) -> anyhow::Result<Vec<u8>> {
1314
let curl = Command::new("curl")
1415
.args([
1516
"-f", // fail on HTTP errors
@@ -22,15 +23,14 @@ fn download(url: &str) -> anyhow::Result<impl Read + use<>> {
2223
if !output.status.success() {
2324
bail!("curl exited with status {} trying to download {}", output.status, url);
2425
}
25-
Ok(Cursor::new(output.stdout))
26+
Ok(output.stdout)
2627
}
2728

28-
fn unpack_tar_gz(content: impl Read, path: &str) -> anyhow::Result<Vec<u8>> {
29+
fn unpack_tar_gz(tarball: impl Read, path: &str) -> anyhow::Result<Vec<u8>> {
2930
use flate2::read::GzDecoder;
3031
use tar::Archive;
3132

32-
// let path = path.as_ref();
33-
let tar = GzDecoder::new(content);
33+
let tar = GzDecoder::new(tarball);
3434
let mut archive = Archive::new(tar);
3535
for entry in archive.entries()? {
3636
let mut entry = entry?;
@@ -43,89 +43,124 @@ fn unpack_tar_gz(content: impl Read, path: &str) -> anyhow::Result<Vec<u8>> {
4343
bail!("Path {path} not found in tar gz")
4444
}
4545

46-
fn download_and_unpack_tar_gz(url: &str, path: &str) -> anyhow::Result<Vec<u8>> {
47-
let resp = download(url).context(format!("Failed to get ok response from {url}"))?;
48-
let data = unpack_tar_gz(resp, path)
49-
.context(format!("Failed to download or unpack {path} out of {url}"))?;
50-
Ok(data)
46+
fn sha256_hex(bytes: &[u8]) -> String {
47+
let digest = Sha256::digest(bytes);
48+
let mut s = String::with_capacity(64);
49+
for b in digest {
50+
write!(&mut s, "{b:02x}").unwrap();
51+
}
52+
s
5153
}
5254

53-
/// (url, `path_in_targz`, `expected_hash`)
54-
type BinaryDownload = (&'static str, &'static str, u128);
55+
struct BinaryDownload {
56+
/// Identifier used both as the on-disk filename in `OUT_DIR` and as the
57+
/// env-var prefix consumed by `artifact!($name)` at runtime.
58+
name: &'static str,
59+
/// GitHub release asset URL.
60+
url: &'static str,
61+
/// Path of the binary within the tarball.
62+
path_in_targz: &'static str,
63+
/// SHA-256 of the extracted binary. Doubles as the cache key: an
64+
/// already-extracted binary in `OUT_DIR` whose content hashes to this
65+
/// value is reused without hitting the network.
66+
expected_sha256: &'static str,
67+
}
5568

5669
const MACOS_BINARY_DOWNLOADS: &[(&str, &[BinaryDownload])] = &[
5770
(
5871
"aarch64",
5972
&[
60-
(
61-
"https://github.com/branchseer/oils-for-unix-build/releases/download/oils-for-unix-0.37.0/oils-for-unix-0.37.0-darwin-arm64.tar.gz",
62-
"oils-for-unix",
63-
282_073_174_065_923_237_490_435_663_309_538_399_576,
64-
),
65-
(
66-
"https://github.com/uutils/coreutils/releases/download/0.4.0/coreutils-0.4.0-aarch64-apple-darwin.tar.gz",
67-
"coreutils-0.4.0-aarch64-apple-darwin/coreutils",
68-
35_998_406_686_137_668_997_937_014_088_186_935_383,
69-
),
73+
// https://github.com/branchseer/oils-for-unix-build/releases/tag/oils-for-unix-0.37.0
74+
BinaryDownload {
75+
name: "oils_for_unix",
76+
url: "https://github.com/branchseer/oils-for-unix-build/releases/download/oils-for-unix-0.37.0/oils-for-unix-0.37.0-darwin-arm64.tar.gz",
77+
path_in_targz: "oils-for-unix",
78+
expected_sha256: "ce4bb80b15f0a0371af08b19b65bfa5ea17d30429ebb911f487de3d2bcc7a07d",
79+
},
80+
// https://github.com/uutils/coreutils/releases/tag/0.4.0
81+
BinaryDownload {
82+
name: "coreutils",
83+
url: "https://github.com/uutils/coreutils/releases/download/0.4.0/coreutils-0.4.0-aarch64-apple-darwin.tar.gz",
84+
path_in_targz: "coreutils-0.4.0-aarch64-apple-darwin/coreutils",
85+
expected_sha256: "8e8f38d9323135a19a73d617336fce85380f3c46fcb83d3ae3e031d1c0372f21",
86+
},
7087
],
7188
),
7289
(
7390
"x86_64",
7491
&[
75-
(
76-
"https://github.com/branchseer/oils-for-unix-build/releases/download/oils-for-unix-0.37.0/oils-for-unix-0.37.0-darwin-x86_64.tar.gz",
77-
"oils-for-unix",
78-
142_673_558_272_427_867_831_039_361_796_426_010_330,
79-
),
80-
(
81-
"https://github.com/uutils/coreutils/releases/download/0.4.0/coreutils-0.4.0-x86_64-apple-darwin.tar.gz",
82-
"coreutils-0.4.0-x86_64-apple-darwin/coreutils",
83-
120_898_281_113_671_104_995_723_556_995_187_526_689,
84-
),
92+
// https://github.com/branchseer/oils-for-unix-build/releases/tag/oils-for-unix-0.37.0
93+
BinaryDownload {
94+
name: "oils_for_unix",
95+
url: "https://github.com/branchseer/oils-for-unix-build/releases/download/oils-for-unix-0.37.0/oils-for-unix-0.37.0-darwin-x86_64.tar.gz",
96+
path_in_targz: "oils-for-unix",
97+
expected_sha256: "cf1a95993127770e2a5fff277cd256a2bb28cf97d7f83ae42fdccc172cdb540d",
98+
},
99+
// https://github.com/uutils/coreutils/releases/tag/0.4.0
100+
BinaryDownload {
101+
name: "coreutils",
102+
url: "https://github.com/uutils/coreutils/releases/download/0.4.0/coreutils-0.4.0-x86_64-apple-darwin.tar.gz",
103+
path_in_targz: "coreutils-0.4.0-x86_64-apple-darwin/coreutils",
104+
expected_sha256: "6be8bee6e8b91fc44a465203b9cc30538af00084b6657dc136d9e55837753eb1",
105+
},
85106
],
86107
),
87108
];
88109

89-
fn fetch_macos_binaries() -> anyhow::Result<()> {
110+
fn fetch_macos_binaries(out_dir: &Path) -> anyhow::Result<()> {
90111
if env::var("CARGO_CFG_TARGET_OS").unwrap() != "macos" {
91112
return Ok(());
92113
}
93114

94-
let out_dir = current_dir().unwrap().join(Path::new(&std::env::var_os("OUT_DIR").unwrap()));
95-
96115
let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
97116
let downloads = MACOS_BINARY_DOWNLOADS
98117
.iter()
99118
.find(|(arch, _)| *arch == target_arch)
100119
.context(format!("Unsupported macOS arch: {target_arch}"))?
101120
.1;
102-
// let downloads = [(zsh_url.as_str(), "bin/zsh", zsh_hash)];
103-
for (url, path_in_targz, expected_hash) in downloads.iter().copied() {
104-
let filename = path_in_targz.split('/').next_back().unwrap();
105-
let download_path = out_dir.join(filename);
106-
let hash_path = out_dir.join(format!("{filename}.hash"));
107121

108-
let file_exists = matches!(fs::read(&download_path), Ok(existing_file_data) if xxh3_128(&existing_file_data) == expected_hash);
109-
if !file_exists {
110-
let data = download_and_unpack_tar_gz(url, path_in_targz)?;
111-
fs::write(&download_path, &data).context(format!(
112-
"Saving {path_in_targz} in {url} to {}",
113-
download_path.display()
114-
))?;
115-
let actual_hash = xxh3_128(&data);
122+
for BinaryDownload { name, url, path_in_targz, expected_sha256 } in downloads {
123+
let dest = out_dir.join(name);
124+
// Cache hit: an already-extracted binary whose contents hash to
125+
// `expected_sha256` is known-good and reused without redownloading.
126+
let cached = matches!(
127+
fs::read(&dest),
128+
Ok(existing) if sha256_hex(&existing) == *expected_sha256,
129+
);
130+
if !cached {
131+
let tarball = download(url).context(format!("Failed to download {url}"))?;
132+
let data = unpack_tar_gz(Cursor::new(tarball), path_in_targz)
133+
.context(format!("Failed to extract {path_in_targz} from {url}"))?;
134+
let actual_sha256 = sha256_hex(&data);
116135
assert_eq!(
117-
actual_hash, expected_hash,
118-
"expected_hash of {path_in_targz} in {url} needs to be updated"
136+
&actual_sha256, expected_sha256,
137+
"sha256 of {path_in_targz} in {url} does not match — update expected value in MACOS_BINARY_DOWNLOADS",
119138
);
139+
fs::write(&dest, &data).with_context(|| format!("writing {}", dest.display()))?;
120140
}
121-
fs::write(&hash_path, format!("{expected_hash:x}"))?;
141+
materialized_artifact_build::register(name, &dest);
122142
}
123143
Ok(())
124-
// let zsh_path = ensure_downloaded(&zsh_url);
144+
}
145+
146+
fn register_preload_cdylib() -> anyhow::Result<()> {
147+
let env_name = match env::var("CARGO_CFG_TARGET_OS").unwrap().as_str() {
148+
"windows" => "CARGO_CDYLIB_FILE_FSPY_PRELOAD_WINDOWS",
149+
_ if env::var("CARGO_CFG_TARGET_ENV").unwrap() == "musl" => return Ok(()),
150+
_ => "CARGO_CDYLIB_FILE_FSPY_PRELOAD_UNIX",
151+
};
152+
// The cdylib path is content-addressed by cargo; when its content changes
153+
// the path changes. Track it so we re-publish the hash on update.
154+
println!("cargo:rerun-if-env-changed={env_name}");
155+
let dylib_path = env::var_os(env_name).with_context(|| format!("{env_name} not set"))?;
156+
materialized_artifact_build::register("fspy_preload", Path::new(&dylib_path));
157+
Ok(())
125158
}
126159

127160
fn main() -> anyhow::Result<()> {
128161
println!("cargo:rerun-if-changed=build.rs");
129-
fetch_macos_binaries().context("Failed to fetch macOS binaries")?;
162+
let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
163+
fetch_macos_binaries(&out_dir).context("Failed to fetch macOS binaries")?;
164+
register_preload_cdylib().context("Failed to register preload cdylib")?;
130165
Ok(())
131166
}

0 commit comments

Comments
 (0)