diff --git a/Cargo.lock b/Cargo.lock index 63a6bdf9..aa99b2c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -863,6 +863,7 @@ dependencies = [ "tokio", "tracing", "walkdir", + "zccache-artifact", ] [[package]] diff --git a/crates/fbuild-build/Cargo.toml b/crates/fbuild-build/Cargo.toml index 43ad93cc..ae7cf917 100644 --- a/crates/fbuild-build/Cargo.toml +++ b/crates/fbuild-build/Cargo.toml @@ -12,6 +12,7 @@ fbuild-config = { path = "../fbuild-config" } fbuild-paths = { path = "../fbuild-paths" } fbuild-packages = { path = "../fbuild-packages" } fbuild-library-select = { path = "../fbuild-library-select" } +zccache-artifact = { workspace = true } tokio = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/fbuild-build/src/framework_libs.rs b/crates/fbuild-build/src/framework_libs.rs index d5e24e9a..5cdac4d7 100644 --- a/crates/fbuild-build/src/framework_libs.rs +++ b/crates/fbuild-build/src/framework_libs.rs @@ -13,10 +13,13 @@ //! example) stay out of the compile set. See FastLED/fbuild#205. use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use fbuild_library_select::cache::{resolve_cached, CacheKeyInputs}; use fbuild_library_select::resolve as resolve_library_selection; use fbuild_packages::library::FrameworkLibrary; use walkdir::{DirEntry, WalkDir}; +use zccache_artifact::KvStore; /// Resolve framework library source files needed by a project. pub fn resolve_framework_library_sources( @@ -56,6 +59,123 @@ pub fn resolve_framework_library_sources_from_libraries( selection.source_files } +/// Cached counterpart to [`resolve_framework_library_sources`]. +/// +/// Routes the same `(libraries, project_dir, src_dir)` resolution through +/// `fbuild_library_select::cache::resolve_cached` using the supplied +/// `KvStore`. On a backend failure (open, read, write) we log a warning and +/// fall back to the uncached `resolve(...)` so a degraded cache can never +/// poison a build — same philosophy as the corrupt-entry handling already +/// inside `cache.rs`. +pub fn resolve_framework_library_sources_cached( + libraries: &[FrameworkLibrary], + project_dir: &Path, + src_dir: &Path, + key_inputs: &CacheKeyInputs<'_>, + store: &KvStore, +) -> Vec { + let (sources, _hit) = resolve_framework_library_sources_cached_with_hit( + libraries, + project_dir, + src_dir, + key_inputs, + store, + ); + sources +} + +/// Internal helper that returns `(sources, from_cache)` so tests can assert +/// hit/miss without the public API surfacing that bit. The hit flag is +/// `false` whenever the cache backend errored and we fell back to the +/// uncached resolver. +pub(crate) fn resolve_framework_library_sources_cached_with_hit( + libraries: &[FrameworkLibrary], + project_dir: &Path, + src_dir: &Path, + key_inputs: &CacheKeyInputs<'_>, + store: &KvStore, +) -> (Vec, bool) { + let roots = framework_include_scan_roots(project_dir, src_dir); + if libraries.is_empty() { + return (Vec::new(), false); + } + + let seeds = collect_project_seeds(&roots); + let search_paths: Vec = roots.clone(); + + match resolve_cached(&seeds, &search_paths, libraries, key_inputs, store) { + Ok(cached) => { + for name in &cached.selection.required_libraries { + if let Some(lib) = libraries.iter().find(|l| &l.name == name) { + tracing::info!( + "selected framework library '{}': {} source files", + lib.name, + lib.source_files.len() + ); + } + } + tracing::info!( + cache = if cached.from_cache { "hit" } else { "miss" }, + key = %cached.key.to_hex(), + "library-select cache: {}", + if cached.from_cache { "hit" } else { "miss" } + ); + (cached.selection.source_files, cached.from_cache) + } + Err(err) => { + tracing::warn!( + error = %err, + "library-select cache backend error; falling back to uncached resolve" + ); + ( + resolve_framework_library_sources_from_libraries(libraries, &roots), + false, + ) + } + } +} + +/// Process-shared `KvStore` for the library-selection cache. +/// +/// Opens lazily on first call and caches the handle for the rest of the +/// process. Returns `None` on open failure — callers must skip caching +/// (and route through the uncached resolver) rather than crash. +pub fn library_select_kv_store() -> Option<&'static KvStore> { + static STORE: OnceLock> = OnceLock::new(); + STORE + .get_or_init(|| { + let dir = library_select_cache_dir(); + match KvStore::open(&dir) { + Ok(store) => { + tracing::info!( + path = %dir.display(), + "library-select cache: opened KvStore" + ); + Some(store) + } + Err(err) => { + tracing::warn!( + path = %dir.display(), + error = %err, + "library-select cache: failed to open KvStore; \ + resolution will run uncached" + ); + None + } + } + }) + .as_ref() +} + +/// Filesystem location of the library-selection KvStore. +/// +/// Routes through `fbuild_paths::get_cache_root()` so the cache obeys the +/// dev/prod isolation contract (`FBUILD_DEV_MODE=1` → `~/.fbuild/dev/cache`) +/// and any `FBUILD_CACHE_DIR` override. +fn library_select_cache_dir() -> PathBuf { + fbuild_paths::get_cache_root().join("library-selection") +} + /// Project directories to scan for `#include` directives and local headers. pub fn framework_include_scan_roots(project_dir: &Path, src_dir: &Path) -> Vec { let mut roots = Vec::new(); @@ -331,4 +451,54 @@ mod tests { assert_eq!(sources, vec![spi_dir.join("SPI.cpp")]); } + + #[test] + fn cached_resolution_round_trips_through_kvstore() { + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path().join("project"); + let src_dir = project_dir.join("src"); + std::fs::create_dir_all(&src_dir).unwrap(); + std::fs::write(src_dir.join("main.cpp"), "#include \n").unwrap(); + + let spi_dir = tmp.path().join("framework").join("libraries").join("SPI"); + std::fs::create_dir_all(&spi_dir).unwrap(); + std::fs::write(spi_dir.join("SPI.h"), "").unwrap(); + std::fs::write(spi_dir.join("SPI.cpp"), "").unwrap(); + + let libraries = vec![FrameworkLibrary { + name: "SPI".to_string(), + dir: spi_dir.clone(), + include_dirs: vec![spi_dir.clone()], + source_files: vec![spi_dir.join("SPI.cpp")], + }]; + + let framework_root = tmp.path().join("framework"); + let key_inputs = CacheKeyInputs { + toolchain_triple: "test-arm-none-eabi", + framework_install_path: &framework_root, + framework_version: "0.0.0-test", + }; + + let kv = KvStore::open(tmp.path().join("kv")).unwrap(); + + let (first, hit_first) = resolve_framework_library_sources_cached_with_hit( + &libraries, + &project_dir, + &src_dir, + &key_inputs, + &kv, + ); + assert!(!hit_first, "first call must miss the cache"); + assert_eq!(first, vec![spi_dir.join("SPI.cpp")]); + + let (second, hit_second) = resolve_framework_library_sources_cached_with_hit( + &libraries, + &project_dir, + &src_dir, + &key_inputs, + &kv, + ); + assert!(hit_second, "second call must hit the cache"); + assert_eq!(first, second, "cache hit must yield identical sources"); + } } diff --git a/crates/fbuild-build/src/stm32/orchestrator.rs b/crates/fbuild-build/src/stm32/orchestrator.rs index 0da735c1..00db0008 100644 --- a/crates/fbuild-build/src/stm32/orchestrator.rs +++ b/crates/fbuild-build/src/stm32/orchestrator.rs @@ -20,7 +20,10 @@ use fbuild_core::{Platform, Result}; use fbuild_packages::{Framework, Toolchain}; use crate::compile_database::TargetArchitecture; -use crate::framework_libs::resolve_framework_library_sources; +use crate::framework_libs::{ + library_select_kv_store, resolve_framework_library_sources, + resolve_framework_library_sources_cached, +}; use crate::generic_arm::{ArmCompiler, ArmLinker}; use crate::pipeline; use crate::source_scanner::SourceCollection; @@ -110,8 +113,34 @@ impl BuildOrchestrator for Stm32Orchestrator { // STM32duino only exposes bundled libraries via this framework-level // discovery (PlatformIO's LDF does the same for `framework = arduino`). let framework_libs = framework.get_framework_libraries(); - let framework_library_sources = - resolve_framework_library_sources(&framework_libs, ¶ms.project_dir, &ctx.src_dir); + // WHY: STM32duino targets every Cortex-M family from M0 (F0xx) up + // through M7 (H7xx) but the toolchain triple is constant + // (`arm-none-eabi`). The cache key already includes + // `framework_install_path` + `framework_version`, so per-MCU drift + // is handled there — this string only needs to disambiguate stm32 + // from teensy etc. so cross-platform key collisions are impossible. + let framework_info = fbuild_packages::Package::get_info(&framework); + let framework_library_sources = match library_select_kv_store() { + Some(store) => { + let key_inputs = fbuild_library_select::cache::CacheKeyInputs { + toolchain_triple: "stm32-arm-none-eabi", + framework_install_path: &framework_info.install_path, + framework_version: &framework_info.version, + }; + resolve_framework_library_sources_cached( + &framework_libs, + ¶ms.project_dir, + &ctx.src_dir, + &key_inputs, + store, + ) + } + None => resolve_framework_library_sources( + &framework_libs, + ¶ms.project_dir, + &ctx.src_dir, + ), + }; if !framework_library_sources.is_empty() { tracing::info!( "STM32 framework library sources added: {}", diff --git a/crates/fbuild-build/src/teensy/orchestrator.rs b/crates/fbuild-build/src/teensy/orchestrator.rs index f3854e8f..211fb893 100644 --- a/crates/fbuild-build/src/teensy/orchestrator.rs +++ b/crates/fbuild-build/src/teensy/orchestrator.rs @@ -24,7 +24,10 @@ use crate::build_fingerprint::{ }; use crate::compile_database::TargetArchitecture; use crate::compiler::Compiler as _; -use crate::framework_libs::resolve_framework_library_sources; +use crate::framework_libs::{ + library_select_kv_store, resolve_framework_library_sources, + resolve_framework_library_sources_cached, +}; use crate::pipeline; use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; @@ -159,8 +162,33 @@ impl BuildOrchestrator for TeensyOrchestrator { .retain(|p| p.file_name().map(|f| f != "Blink.cc").unwrap_or(true)); let framework_libs = framework.get_framework_libraries(); - let framework_library_sources = - resolve_framework_library_sources(&framework_libs, ¶ms.project_dir, &ctx.src_dir); + // WHY: Teensy 3.x/4.x and TeensyLC all share teensyduino's + // arm-none-eabi toolchain — a single stable triple covers every + // board this orchestrator handles. The triple feeds the cache key + // so bumping it invalidates the entire teensy slice without + // touching SCANNER_VERSION / LDF_MODE_VERSION. + let framework_info = fbuild_packages::Package::get_info(&framework); + let framework_library_sources = match library_select_kv_store() { + Some(store) => { + let key_inputs = fbuild_library_select::cache::CacheKeyInputs { + toolchain_triple: "teensy-arm-none-eabi", + framework_install_path: &framework_info.install_path, + framework_version: &framework_info.version, + }; + resolve_framework_library_sources_cached( + &framework_libs, + ¶ms.project_dir, + &ctx.src_dir, + &key_inputs, + store, + ) + } + None => resolve_framework_library_sources( + &framework_libs, + ¶ms.project_dir, + &ctx.src_dir, + ), + }; if !framework_library_sources.is_empty() { tracing::info!( "Teensy framework library sources added: {}", diff --git a/docs/architecture/library-selection.md b/docs/architecture/library-selection.md index aefa687e..2cb6f6ab 100644 --- a/docs/architecture/library-selection.md +++ b/docs/architecture/library-selection.md @@ -2,8 +2,9 @@ > Status: foundation phases (0–3 + Phase 5 framework_libs delegation) landed > in PR #207. Phase 6 acceptance gates and Phase 8.a `lib-select` CLI landed -> in PR #208. Phase 4 (zccache memoization) tracked at zackees/zccache#130. -> Phase 7 perf gates and Phase 8.b cleanup remain follow-ups in `#205`. +> in PR #208. Phase 4 (zccache memoization, `resolve_cached`) shipped in +> PR #212 and is wired into the teensy + stm32 orchestrators by #214. +> Phase 7 perf gates remain a follow-up in `#205`. ## Why @@ -93,22 +94,25 @@ original issue framing ("fixed-point over include closure — typically 2–3 iterations") was wrong; we match PIO's 2-pass semantics exactly so users who flip between PlatformIO and fbuild see the same library set. -## Cache key (Phase 4, not yet shipped) +## Cache key -The resolver output is a pure function of: +`resolve_cached` (see `crates/fbuild-library-select/src/cache.rs`) hashes: - sorted blake3s of project source content, -- sorted blake3s of each lib's canonical headers + `library.json` / - `library.properties`, -- ordered search-path list, +- sorted blake3s of each lib's canonical header + (`/.h`), +- ordered search-path list (order matters — PIO's resolution is + order-sensitive), - toolchain triple, - framework install path + version identifier, - `SCANNER_VERSION` (bumped on tokenizer changes), - `LDF_MODE_VERSION` (bumped on resolver semantic changes). -Memoization is gated on the K/V proposal at zackees/zccache#130 -(`tasks/zccache-kv-design.md`). The resolver is already deterministic and -sort-stable, so cache wiring is a pure addition with no behavior change. +The KvStore is opened lazily by `framework_libs::library_select_kv_store()` +under `fbuild_paths::get_cache_root().join("library-selection")`, which +respects `FBUILD_DEV_MODE` and `FBUILD_CACHE_DIR`. Wiring the cache was a +pure addition: orchestrators that hit a backend error fall through to the +uncached `resolve(...)` rather than fail. ## Determinism @@ -137,16 +141,16 @@ keys safe. ## Future work -- **Phase 4** — zccache K/V memoization. Gated on zackees/zccache#130 - shipping a versioned `KvStore` API and a 1.4.0 release; see - `tasks/zccache-kv-design.md`. -- **Phase 7** — perf gates wired into `bench/fastled-examples`. -- **Phase 8.b** — final deletion of any dead helpers in `framework_libs.rs` - once Phase 4 cache lands. +- **Phase 7** — perf gates wired into `bench/fastled-examples` to catch + regressions in both cold-resolve and warm cache-hit paths (#215). +- **Phase 8.b** — `framework_libs.rs` is now a thin delegator: scan-root + collection, the uncached and cached entry points, the `KvStore` opener, + and tests. No remaining dead helpers to retire. ## References - PlatformIO LDF source: `platformio/builder/tools/piolib.py`. - Issue: FastLED/fbuild#205. - Closes: FastLED/fbuild#202, FastLED/fbuild#204. -- Cache prerequisite: zackees/zccache#130. +- Cache wiring: FastLED/fbuild#212 (cache helper), FastLED/fbuild#214 + (orchestrator wiring).