diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 78bb5a433..a41af5b8c 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -877,7 +877,10 @@ where let dist_client = match dist_compile_cmd.clone().and(dist_client) { Some(dc) => dc, None => { - debug!("[{}]: Compiling locally", out_pretty); + info!( + "[{}]: Compiling locally (not eligible for distributed compilation)", + out_pretty + ); return compile_cmd .execute(service, &creator) .await @@ -885,7 +888,7 @@ where } }; - debug!("[{}]: Attempting distributed compilation", out_pretty); + info!("[{}]: Attempting distributed compilation", out_pretty); let out_pretty2 = out_pretty.clone(); let local_executable = compile_cmd.get_executable(); @@ -900,6 +903,15 @@ where .map(|output| path_transformer.as_dist_abs(&cwd.join(output.path))) .collect::>() .context("Failed to adapt an output path for distributed compile")?; + // The local paths every declared output must occupy once the compile + // finishes. Captured before `compilation` is moved into the packagers + // below so a remote compile that drops a declared output can be detected + // and salvaged (see the missing-output check after the run completes). + let expected_output_paths: Vec = compilation + .outputs() + .filter(|output| !output.optional) + .map(|output| cwd.join(output.path)) + .collect(); let (inputs_packager, toolchain_packager, outputs_rewriter) = compilation.into_dist_packagers(path_transformer)?; @@ -970,7 +982,7 @@ where ) })?; - let mut jc = match jres { + let jc = match jres { dist::RunJobResult::Complete(jc) => jc, dist::RunJobResult::JobNotFound => bail!("Job {} not found on server", job_id), }; @@ -1036,14 +1048,56 @@ where ); if jc.output.code != 0 { - // Add server info to help diagnose host-specific failures, e.g. due to flaky hardware. - // Failed builds are not cached so this tampering should not cause too much trouble. - let server_info = format!("sccache: Job failed on server {}:\n", server_id.addr()); - jc.output - .stderr - .splice(0..0, server_info.as_bytes().to_vec()); + // The one-line reason above is logged by the caller at warn; the + // remote compiler's own stdout/stderr is the only place the actual + // failure (a missing target std, an unshipped input) is visible, and + // the local fallback discards it. Emit it at debug so SCCACHE_LOG=debug + // surfaces the remote error without a rebuild, while a normal + // SCCACHE_LOG=info run stays quiet. + debug!( + "[{}]: distributed compile on {} exited {}; remote stderr:\n{}\nremote stdout:\n{}", + out_pretty, + server_id.addr(), + jc.output.code, + String::from_utf8_lossy(&jc.output.stderr), + String::from_utf8_lossy(&jc.output.stdout), + ); + // A non-zero remote result is frequently a distribution artifact + // rather than a genuine compiler error: e.g. an object that + // .incbin's a binary the inputs packager did not ship (the kernel's + // vdso/dtb/embedded-config wrappers), which the build-server cannot + // assemble. Fall back to a local recompile via the or_else below - it + // either succeeds (confirming a dist-only artifact) or reproduces the + // real error locally, so a remote failure never breaks a build that + // would compile fine locally. This only affects failing dist + // compiles; successful ones are returned unchanged above. + bail!( + "distributed compile on {} returned exit code {}; recompiling locally", + server_id.addr(), + jc.output.code + ); + } + + // The remote compile returned exit 0 but may have dropped a declared + // output: glibc's ldconfig.o/sprof.o omit their `.o.dt` dep file, which + // the build-server does not return. Zipping the outputs later would then + // fail fatally with no recourse. Treat a missing declared output as a + // distribution artifact and fall back to a local recompile via the + // or_else below, which reproduces the full output set. Compiles whose + // dist output set is complete are unaffected. + if let Some(missing) = expected_output_paths.iter().find(|p| !p.exists()) { + bail!( + "distributed compile on {} did not return expected output {}; recompiling locally", + server_id.addr(), + missing.display() + ); } + info!( + "[{}]: Distributed compilation finished on {}", + out_pretty, + server_id.addr() + ); Ok((DistType::Ok(server_id), jc.output.into())) }; @@ -3209,6 +3263,7 @@ LLVM version: 6.0", test_dist::ErrorAllocJobClient::new(), test_dist::ErrorSubmitToolchainClient::new(), test_dist::ErrorRunJobClient::new(), + test_dist::IncompleteOutputsClient::new(), ]; // Write a dummy input file so the preprocessor cache mode can work std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); @@ -3677,4 +3732,112 @@ mod test_dist { None } } + + /// A dist client whose remote compile succeeds (exit 0) but whose returned + /// output set drops a declared output - the glibc `.o.dt` failure mode, + /// where the build-server runs the compile fine yet does not return every + /// declared output. The client must detect the missing output and fall back + /// to a local recompile rather than failing the build. + pub struct IncompleteOutputsClient { + has_started: AtomicBool, + tc: Toolchain, + output: ProcessOutput, + } + + impl IncompleteOutputsClient { + #[allow(clippy::new_ret_no_self)] + pub fn new() -> Arc { + Arc::new(Self { + has_started: AtomicBool::default(), + tc: Toolchain { + archive_id: "somearchiveid".to_owned(), + }, + output: ProcessOutput::fake_output(0, vec![], vec![]), + }) + } + } + + #[async_trait] + impl dist::Client for IncompleteOutputsClient { + async fn do_alloc_job(&self, tc: Toolchain) -> Result { + assert!( + !self + .has_started + .swap(true, std::sync::atomic::Ordering::AcqRel) + ); + assert_eq!(self.tc, tc); + + Ok(AllocJobResult::Success { + job_alloc: JobAlloc { + auth: "abcd".to_owned(), + job_id: JobId(0), + server_id: ServerId::new(([0, 0, 0, 0], 1).into()), + }, + need_toolchain: true, + }) + } + async fn do_get_status(&self) -> Result { + unreachable!("fn do_get_status is not used for this test. qed") + } + async fn do_submit_toolchain( + &self, + job_alloc: JobAlloc, + tc: Toolchain, + ) -> Result { + assert_eq!(job_alloc.job_id, JobId(0)); + assert_eq!(self.tc, tc); + + Ok(SubmitToolchainResult::Success) + } + async fn do_run_job( + &self, + job_alloc: JobAlloc, + command: CompileCommand, + outputs: Vec, + inputs_packager: Box, + ) -> Result<(RunJobResult, PathTransformer)> { + assert_eq!(job_alloc.job_id, JobId(0)); + assert_eq!(command.executable, "/overridden/compiler"); + + let mut inputs = vec![]; + let path_transformer = inputs_packager.write_inputs(&mut inputs).unwrap(); + // Drop one declared output to mimic a build-server that returned a + // successful compile with an incomplete output set. + let mut outputs = outputs; + outputs.pop(); + let outputs = outputs + .into_iter() + .map(|name| { + let data = format!("some data in {}", name); + let data = OutputData::try_from_reader(data.as_bytes()).unwrap(); + (name, data) + }) + .collect(); + let result = RunJobResult::Complete(JobComplete { + output: self.output.clone(), + outputs, + }); + Ok((result, path_transformer)) + } + async fn put_toolchain( + &self, + _: PathBuf, + _: String, + _: Box, + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + Ok(( + self.tc.clone(), + Some(( + "/overridden/compiler".to_owned(), + PathBuf::from("somearchiveid"), + )), + )) + } + fn rewrite_includes_only(&self) -> bool { + false + } + fn get_custom_toolchain(&self, _exe: &Path) -> Option { + None + } + } } diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index d6c2c7192..d5549a26c 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -1622,7 +1622,13 @@ where dep_info.to_string_lossy().into_owned(), ArtifactDescriptor { path: p.clone(), - optional: false, + // The build server does not reliably return the rust + // dep-info (.d) file, and it only feeds cargo's rebuild + // tracking (a bitbake do_compile runs cargo from scratch and + // never consumes it). Mark it optional so a distributed + // compile that omits it is not discarded and recompiled + // locally - which stranded every rust compile on the client. + optional: true, }, ); Some(p) diff --git a/src/lib.rs b/src/lib.rs index d5aa6a603..f6084803e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -62,8 +62,6 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const LOGGING_ENV: &str = "SCCACHE_LOG"; pub fn main() { - init_logging(); - let command = match cmdline::try_parse() { Ok(cmd) => cmd, Err(e) => match e.downcast::() { @@ -79,6 +77,10 @@ pub fn main() { }, }; + // Logging is initialized after parsing so the compile-wrapper case can keep + // sccache's own log records off the wrapped compiler's stderr. + init_logging(&command); + std::process::exit(match commands::run_command(command) { Ok(s) => s, Err(e) => { @@ -91,7 +93,7 @@ pub fn main() { }); } -fn init_logging() { +fn init_logging(command: &cmdline::Command) { if env::var(LOGGING_ENV).is_ok() { let mut builder = env_logger::Builder::from_env(LOGGING_ENV); @@ -100,6 +102,35 @@ fn init_logging() { builder.format_timestamp_millis(); } + // When sccache runs as a compiler wrapper (`sccache ...`) it + // shares stderr with the wrapped compiler. Build tools treat any + // unexpected stderr on a feature probe as a compiler failure: libtool's + // `-fPIC` check sets `pic_flag=""` on non-empty stderr, producing non-PIC + // objects that then fail to link into a shared library. sccache's own log + // records must never reach that stderr. Send them to SCCACHE_ERROR_LOG + // when set; otherwise keep stderr only for an interactive terminal (so + // `SCCACHE_LOG=debug sccache gcc ...` at a prompt still shows logs) and + // discard them when stderr is captured by a build. + if matches!(command, cmdline::Command::Compile { .. }) { + use std::io::IsTerminal; + let target: Option> = + match env::var("SCCACHE_ERROR_LOG") { + Ok(path) if !path.is_empty() => Some( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map(|f| Box::new(f) as Box) + .unwrap_or_else(|_| Box::new(std::io::sink())), + ), + _ if !std::io::stderr().is_terminal() => Some(Box::new(std::io::sink())), + _ => None, + }; + if let Some(target) = target { + builder.target(env_logger::Target::Pipe(target)); + } + } + match builder.try_init() { Ok(_) => (), Err(e) => panic!("Failed to initialize logging: {:?}", e),