From 7db4201a7cb2602aa5a4d8595fa6b7b749159345 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 23 Jun 2026 14:09:55 -0600 Subject: [PATCH 1/6] compiler: log distributed-compile decisions at info level sccache logged the distribute-vs-local decision only at debug ("Compiling locally", "Attempting distributed compilation"), while an infrastructure fallback warned. At info a successful distribution and a local compile were both silent, so the only visible dist signal was the failure path - leaving no way to see the distribute/fallback ratio without the full debug firehose. Diagnosing why a distributed build under-distributes meant guessing. Promote the decision points to info and add a log on successful distribution naming the server and exit code, so SCCACHE_LOG=info gives a per-compile dist trace (attempt then distributed-on-server, compiled-locally, or falling-back-with-reason). sccache only emits logs when SCCACHE_LOG is set, so default runs are unaffected. Signed-off-by: Javier Tia (cherry picked from commit 950ddfdb3e801ecda2d07ade4f6651ffacc750cb) --- src/compiler/compiler.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 78bb5a433..77eea183e 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -877,7 +877,10 @@ where let dist_client = match dist_compile_cmd.clone().and(dist_client) { Some(dc) => dc, None => { - debug!("[{}]: Compiling locally", out_pretty); + info!( + "[{}]: Compiling locally (not eligible for distributed compilation)", + out_pretty + ); return compile_cmd .execute(service, &creator) .await @@ -885,7 +888,7 @@ where } }; - debug!("[{}]: Attempting distributed compilation", out_pretty); + info!("[{}]: Attempting distributed compilation", out_pretty); let out_pretty2 = out_pretty.clone(); let local_executable = compile_cmd.get_executable(); @@ -1044,6 +1047,12 @@ where .splice(0..0, server_info.as_bytes().to_vec()); } + info!( + "[{}]: Distributed compilation finished on {} (exit code {})", + out_pretty, + server_id.addr(), + jc.output.code + ); Ok((DistType::Ok(server_id), jc.output.into())) }; From f2c99e723874d5342e448202ffc3191835f2b9b7 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 23 Jun 2026 15:03:30 -0600 Subject: [PATCH 2/6] compiler: fall back to local on distributed-compile failure A distributed compile the build-server rejects is often a distribution artifact, not a genuine compiler error: an object that .incbin's a binary the inputs packager does not ship - the kernel's vdso, embedded-config, and dtb wrappers - cannot be assembled remotely and returns non-zero, which failed the whole build with no recourse and forced the kernel to be excluded wholesale. Treat a non-zero remote result as a fallback trigger rather than a terminal error: recompile locally, which either succeeds (confirming a dist-only artifact) or reproduces the genuine error. A remote failure can no longer break a build that would compile locally. Only failing dist compiles are affected - successful ones are returned unchanged - so the kernel now distributes (1124/1128 compiles), with its handful of .incbin objects falling back to local. Signed-off-by: Javier Tia (cherry picked from commit b53367ce3083f7ca52861f84a89bbe530c5e80df) --- src/compiler/compiler.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 77eea183e..d3ceee25a 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -973,7 +973,7 @@ where ) })?; - let mut jc = match jres { + let jc = match jres { dist::RunJobResult::Complete(jc) => jc, dist::RunJobResult::JobNotFound => bail!("Job {} not found on server", job_id), }; @@ -1039,19 +1039,26 @@ where ); if jc.output.code != 0 { - // Add server info to help diagnose host-specific failures, e.g. due to flaky hardware. - // Failed builds are not cached so this tampering should not cause too much trouble. - let server_info = format!("sccache: Job failed on server {}:\n", server_id.addr()); - jc.output - .stderr - .splice(0..0, server_info.as_bytes().to_vec()); + // A non-zero remote result is frequently a distribution artifact + // rather than a genuine compiler error: e.g. an object that + // .incbin's a binary the inputs packager did not ship (the kernel's + // vdso/dtb/embedded-config wrappers), which the build-server cannot + // assemble. Fall back to a local recompile via the or_else below - it + // either succeeds (confirming a dist-only artifact) or reproduces the + // real error locally, so a remote failure never breaks a build that + // would compile fine locally. This only affects failing dist + // compiles; successful ones are returned unchanged above. + bail!( + "distributed compile on {} returned exit code {}; recompiling locally", + server_id.addr(), + jc.output.code + ); } info!( - "[{}]: Distributed compilation finished on {} (exit code {})", + "[{}]: Distributed compilation finished on {}", out_pretty, - server_id.addr(), - jc.output.code + server_id.addr() ); Ok((DistType::Ok(server_id), jc.output.into())) }; From a30e8a43a7f2a68dfebd69d4d5bd72d180ebb383 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Wed, 24 Jun 2026 16:50:12 -0600 Subject: [PATCH 3/6] compiler: fall back to local when a dist compile drops an output A distributed compile can return a successful (exit 0) result yet omit a declared output. glibc's ldconfig.o and sprof.o compile fine on the build-server, but it does not return their `.o.dt` dependency file, so zipping the compiler outputs fails fatally ("failed to open file ...o.dt: No such file") with no recourse. One dropped output among 6427 forced glibc to be excluded from distribution wholesale, even though 6425 of its compiles distribute cleanly. Capture the declared output paths before the compilation is moved into the packagers, and after a successful remote compile verify each one exists on disk. A missing output is a distribution artifact, not a compiler error, so bail into the existing local-recompile fallback (the same one that already salvages non-zero remote results), which reproduces the full output set. Only compiles whose dist output set is incomplete are affected; complete ones are returned unchanged. glibc now distributes, with ldconfig.o and sprof.o falling back to local. Signed-off-by: Javier Tia (cherry picked from commit 3b365f3dfa596c4b50815ac4424ee796503232f6) --- src/compiler/compiler.rs | 132 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index d3ceee25a..eb5ec5195 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -903,6 +903,14 @@ where .map(|output| path_transformer.as_dist_abs(&cwd.join(output.path))) .collect::>() .context("Failed to adapt an output path for distributed compile")?; + // The local paths every declared output must occupy once the compile + // finishes. Captured before `compilation` is moved into the packagers + // below so a remote compile that drops a declared output can be detected + // and salvaged (see the missing-output check after the run completes). + let expected_output_paths: Vec = compilation + .outputs() + .map(|output| cwd.join(output.path)) + .collect(); let (inputs_packager, toolchain_packager, outputs_rewriter) = compilation.into_dist_packagers(path_transformer)?; @@ -1055,6 +1063,21 @@ where ); } + // The remote compile returned exit 0 but may have dropped a declared + // output: glibc's ldconfig.o/sprof.o omit their `.o.dt` dep file, which + // the build-server does not return. Zipping the outputs later would then + // fail fatally with no recourse. Treat a missing declared output as a + // distribution artifact and fall back to a local recompile via the + // or_else below, which reproduces the full output set. Compiles whose + // dist output set is complete are unaffected. + if let Some(missing) = expected_output_paths.iter().find(|p| !p.exists()) { + bail!( + "distributed compile on {} did not return expected output {}; recompiling locally", + server_id.addr(), + missing.display() + ); + } + info!( "[{}]: Distributed compilation finished on {}", out_pretty, @@ -3225,6 +3248,7 @@ LLVM version: 6.0", test_dist::ErrorAllocJobClient::new(), test_dist::ErrorSubmitToolchainClient::new(), test_dist::ErrorRunJobClient::new(), + test_dist::IncompleteOutputsClient::new(), ]; // Write a dummy input file so the preprocessor cache mode can work std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); @@ -3693,4 +3717,112 @@ mod test_dist { None } } + + /// A dist client whose remote compile succeeds (exit 0) but whose returned + /// output set drops a declared output - the glibc `.o.dt` failure mode, + /// where the build-server runs the compile fine yet does not return every + /// declared output. The client must detect the missing output and fall back + /// to a local recompile rather than failing the build. + pub struct IncompleteOutputsClient { + has_started: AtomicBool, + tc: Toolchain, + output: ProcessOutput, + } + + impl IncompleteOutputsClient { + #[allow(clippy::new_ret_no_self)] + pub fn new() -> Arc { + Arc::new(Self { + has_started: AtomicBool::default(), + tc: Toolchain { + archive_id: "somearchiveid".to_owned(), + }, + output: ProcessOutput::fake_output(0, vec![], vec![]), + }) + } + } + + #[async_trait] + impl dist::Client for IncompleteOutputsClient { + async fn do_alloc_job(&self, tc: Toolchain) -> Result { + assert!( + !self + .has_started + .swap(true, std::sync::atomic::Ordering::AcqRel) + ); + assert_eq!(self.tc, tc); + + Ok(AllocJobResult::Success { + job_alloc: JobAlloc { + auth: "abcd".to_owned(), + job_id: JobId(0), + server_id: ServerId::new(([0, 0, 0, 0], 1).into()), + }, + need_toolchain: true, + }) + } + async fn do_get_status(&self) -> Result { + unreachable!("fn do_get_status is not used for this test. qed") + } + async fn do_submit_toolchain( + &self, + job_alloc: JobAlloc, + tc: Toolchain, + ) -> Result { + assert_eq!(job_alloc.job_id, JobId(0)); + assert_eq!(self.tc, tc); + + Ok(SubmitToolchainResult::Success) + } + async fn do_run_job( + &self, + job_alloc: JobAlloc, + command: CompileCommand, + outputs: Vec, + inputs_packager: Box, + ) -> Result<(RunJobResult, PathTransformer)> { + assert_eq!(job_alloc.job_id, JobId(0)); + assert_eq!(command.executable, "/overridden/compiler"); + + let mut inputs = vec![]; + let path_transformer = inputs_packager.write_inputs(&mut inputs).unwrap(); + // Drop one declared output to mimic a build-server that returned a + // successful compile with an incomplete output set. + let mut outputs = outputs; + outputs.pop(); + let outputs = outputs + .into_iter() + .map(|name| { + let data = format!("some data in {}", name); + let data = OutputData::try_from_reader(data.as_bytes()).unwrap(); + (name, data) + }) + .collect(); + let result = RunJobResult::Complete(JobComplete { + output: self.output.clone(), + outputs, + }); + Ok((result, path_transformer)) + } + async fn put_toolchain( + &self, + _: PathBuf, + _: String, + _: Box, + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + Ok(( + self.tc.clone(), + Some(( + "/overridden/compiler".to_owned(), + PathBuf::from("somearchiveid"), + )), + )) + } + fn rewrite_includes_only(&self) -> bool { + false + } + fn get_custom_toolchain(&self, _exe: &Path) -> Option { + None + } + } } From 6654ea01c8d607d7d0802718129f8d7b37550325 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Thu, 25 Jun 2026 17:01:38 -0600 Subject: [PATCH 4/6] logging: keep wrapper log output off the compiler's stderr When sccache runs as a compiler wrapper (`sccache ...`) it shares stderr with the wrapped compiler. A build's feature probes treat any unexpected stderr as a compiler failure: libtool's -fPIC check (_LT_COMPILER_OPTION) sets pic_flag="" whenever the probe compile emits stderr, even on exit 0. With SCCACHE_LOG set, the client's own env_logger records ("Attempting to read config file", "Server sent CompileStarted") landed on that shared stderr, so an autotools package configured with pic_flag="" and produced non-PIC objects that failed to link into a shared library (relocation R_X86_64_32 ... recompile with -fPIC). It looked concurrency-dependent because the client's startup logging varies with server-connection timing, so the probe stderr only sometimes diverged from libtool's expected boilerplate. Initialize logging after parsing the command so the compile-wrapper case can choose its target: route sccache's own records to SCCACHE_ERROR_LOG when set, keep stderr only when it is an interactive terminal (so `SCCACHE_LOG=debug sccache gcc ...` at a prompt still shows logs), and discard them when stderr is captured by a build. The wrapped compiler's forwarded stderr is untouched. Signed-off-by: Javier Tia (cherry picked from commit 63df688bd154dcc97a8b1a8f809d8cc92a952042) --- src/lib.rs | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d5aa6a603..f6084803e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -62,8 +62,6 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const LOGGING_ENV: &str = "SCCACHE_LOG"; pub fn main() { - init_logging(); - let command = match cmdline::try_parse() { Ok(cmd) => cmd, Err(e) => match e.downcast::() { @@ -79,6 +77,10 @@ pub fn main() { }, }; + // Logging is initialized after parsing so the compile-wrapper case can keep + // sccache's own log records off the wrapped compiler's stderr. + init_logging(&command); + std::process::exit(match commands::run_command(command) { Ok(s) => s, Err(e) => { @@ -91,7 +93,7 @@ pub fn main() { }); } -fn init_logging() { +fn init_logging(command: &cmdline::Command) { if env::var(LOGGING_ENV).is_ok() { let mut builder = env_logger::Builder::from_env(LOGGING_ENV); @@ -100,6 +102,35 @@ fn init_logging() { builder.format_timestamp_millis(); } + // When sccache runs as a compiler wrapper (`sccache ...`) it + // shares stderr with the wrapped compiler. Build tools treat any + // unexpected stderr on a feature probe as a compiler failure: libtool's + // `-fPIC` check sets `pic_flag=""` on non-empty stderr, producing non-PIC + // objects that then fail to link into a shared library. sccache's own log + // records must never reach that stderr. Send them to SCCACHE_ERROR_LOG + // when set; otherwise keep stderr only for an interactive terminal (so + // `SCCACHE_LOG=debug sccache gcc ...` at a prompt still shows logs) and + // discard them when stderr is captured by a build. + if matches!(command, cmdline::Command::Compile { .. }) { + use std::io::IsTerminal; + let target: Option> = + match env::var("SCCACHE_ERROR_LOG") { + Ok(path) if !path.is_empty() => Some( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map(|f| Box::new(f) as Box) + .unwrap_or_else(|_| Box::new(std::io::sink())), + ), + _ if !std::io::stderr().is_terminal() => Some(Box::new(std::io::sink())), + _ => None, + }; + if let Some(target) = target { + builder.target(env_logger::Target::Pipe(target)); + } + } + match builder.try_init() { Ok(_) => (), Err(e) => panic!("Failed to initialize logging: {:?}", e), From 612871bafffe399e93470547e126310c2e6508b8 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Thu, 2 Jul 2026 10:30:21 -0600 Subject: [PATCH 5/6] dist: don't fail a distributed compile over an optional missing output Softening the rust dep-info rewriter (previous commit) was not enough to make rust distribute: a rust compile still fell back to local, now caught by the second guard. After a distributed compile the client verifies every declared output exists and, if one is missing, discards the result and recompiles locally. That guard ignored the `optional` flag, so a missing optional output forced a local recompile even though the output was declared droppable - stranding every rust compile on the client because the rust dep-info (.d) file, which the build server does not reliably return, was declared non-optional. Make the guard honor `optional`: only non-optional outputs must exist. The C dep file and glibc's dropped .o.dt stay non-optional, so their local-fallback (the reason that guard exists) is unchanged; a `-gsplit-dwarf` .dwo, already declared optional, no longer wrongly forces fallback. Then mark the rust .d optional, since it only feeds cargo rebuild tracking a from-scratch bitbake do_compile never consumes. With both, a distributed rust compile whose .d is absent keeps its object and rlib instead of recompiling locally. Signed-off-by: Javier Tia (cherry picked from commit 0ac0fe17badabb93fd8b22e348bb3f5995780877) --- src/compiler/compiler.rs | 1 + src/compiler/rust.rs | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index eb5ec5195..0cd8c03fb 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -909,6 +909,7 @@ where // and salvaged (see the missing-output check after the run completes). let expected_output_paths: Vec = compilation .outputs() + .filter(|output| !output.optional) .map(|output| cwd.join(output.path)) .collect(); let (inputs_packager, toolchain_packager, outputs_rewriter) = diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index d6c2c7192..d5549a26c 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -1622,7 +1622,13 @@ where dep_info.to_string_lossy().into_owned(), ArtifactDescriptor { path: p.clone(), - optional: false, + // The build server does not reliably return the rust + // dep-info (.d) file, and it only feeds cargo's rebuild + // tracking (a bitbake do_compile runs cargo from scratch and + // never consumes it). Mark it optional so a distributed + // compile that omits it is not discarded and recompiled + // locally - which stranded every rust compile on the client. + optional: true, }, ); Some(p) From 974dbe893b9ab5fbde48fcedba623664caaa71b8 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Thu, 2 Jul 2026 12:02:21 -0600 Subject: [PATCH 6/6] compiler: log a failed distributed compile's remote output at debug When a distributed compile exits non-zero the client logs only the exit code and falls back to a local recompile, discarding the remote compiler's stdout and stderr. That output is the only place the actual failure is visible - a missing target std, an input the packager did not ship, a sandbox path that does not resolve - so diagnosing a dist regression meant patching in a temporary dump and rebuilding the binary each time. Emit the remote stdout/stderr at debug on the non-zero path. SCCACHE_LOG=debug now surfaces the real remote error with no rebuild, while a normal SCCACHE_LOG=info run stays quiet (the one-line fallback reason is still logged at warn by the caller). Signed-off-by: Javier Tia (cherry picked from commit 1a4182c1db95b777f6e0ffcac11120e0f2730b0d) --- src/compiler/compiler.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 0cd8c03fb..a41af5b8c 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -1048,6 +1048,20 @@ where ); if jc.output.code != 0 { + // The one-line reason above is logged by the caller at warn; the + // remote compiler's own stdout/stderr is the only place the actual + // failure (a missing target std, an unshipped input) is visible, and + // the local fallback discards it. Emit it at debug so SCCACHE_LOG=debug + // surfaces the remote error without a rebuild, while a normal + // SCCACHE_LOG=info run stays quiet. + debug!( + "[{}]: distributed compile on {} exited {}; remote stderr:\n{}\nremote stdout:\n{}", + out_pretty, + server_id.addr(), + jc.output.code, + String::from_utf8_lossy(&jc.output.stderr), + String::from_utf8_lossy(&jc.output.stdout), + ); // A non-zero remote result is frequently a distribution artifact // rather than a genuine compiler error: e.g. an object that // .incbin's a binary the inputs packager did not ship (the kernel's