feat: Fall back to local compilation on server connection/startup failures

Hand-applied port of mozilla#2756 (which is stacked on the
unmerged client-side-mode PR upstream) onto the v0.16.0 base: when the
client cannot connect to or start the background server and
SCCACHE_IGNORE_SERVER_IO_ERROR is set, warn and run the compiler
directly instead of failing the compilation. Without this, a storage
self-check failure at server startup (e.g. an unreachable or
unauthorized cache backend) makes every wrapper invocation exit 2 and
fails the build — an optimization must never do that.

Author: anthonyshew

src/commands.rs

@@ -560,6 +560,23 @@ where
         }
     }
 
+    run_compiler_locally(creator, runtime, exe, cmdline, cwd)
+}
+
+/// Run the compiler directly, without any caching, and return its exit
+/// status. The fallback for invocations the server cannot handle — and,
+/// when `SCCACHE_IGNORE_SERVER_IO_ERROR` is set, for a server that cannot
+/// be reached or started at all.
+fn run_compiler_locally<T>(
+    mut creator: T,
+    runtime: &mut Runtime,
+    exe: &Path,
+    cmdline: Vec<OsString>,
+    cwd: &Path,
+) -> Result<i32>
+where
+    T: CommandCreatorSync,
+{
     let mut cmd = creator.new_command_sync(exe);
     cmd.args(&cmdline).current_dir(cwd);
     if log_enabled!(Trace) {
@@ -807,7 +824,30 @@ pub fn run_command(cmd: Command) -> Result<i32> {
                 });
 
             let jobserver = Client::new();
-            let conn = connect_or_start_server(&get_addr(), startup_timeout)?;
+            let conn = match connect_or_start_server(&get_addr(), startup_timeout) {
+                Ok(conn) => conn,
+                Err(error) if ignore_all_server_io_errors() => {
+                    eprintln!(
+                        "sccache: warning: failed to connect to or start server, compiling \
+                         locally instead: {error:#}"
+                    );
+                    let mut runtime = new_client_runtime()?;
+                    let exe = which_in(
+                        Path::new(&exe),
+                        env::var_os("PATH"),
+                        &cwd,
+                    )?;
+                    return run_compiler_locally(
+                        ProcessCommandCreator::new(&jobserver),
+                        &mut runtime,
+                        &exe,
+                        cmdline,
+                        &cwd,
+                    )
+                    .context("failed to execute compiler locally");
+                }
+                Err(error) => return Err(error),
+            };
             let mut runtime = new_client_runtime()?;
             let res = do_compile(
                 ProcessCommandCreator::new(&jobserver),

tests/sccache_args.rs

@@ -14,6 +14,55 @@ use std::process::Command;
 #[macro_use]
 extern crate log;
 
+#[cfg(unix)]
+fn command_with_server_start_failure() -> Result<(tempfile::TempDir, Command)> {
+    let tempdir = tempfile::tempdir()?;
+    let config = tempdir.path().join("config");
+    std::fs::write(&config, "")?;
+
+    let mut command = Command::new(SCCACHE_BIN.as_os_str());
+    command
+        .args(["rustc", "--version"])
+        .current_dir(tempdir.path())
+        .env("SCCACHE_CONF", config)
+        .env("SCCACHE_DIR", tempdir.path().join("cache"))
+        .env_remove("SCCACHE_IGNORE_SERVER_IO_ERROR")
+        // Force server start to fail with an overlong unix domain socket.
+        .env("SCCACHE_SERVER_UDS", "x".repeat(4096));
+
+    Ok((tempdir, command))
+}
+
+#[cfg(unix)]
+#[test]
+#[serial]
+fn test_server_start_failure_falls_back() -> Result<()> {
+    let (_tempdir, mut command) = command_with_server_start_failure()?;
+
+    command
+        .env("SCCACHE_IGNORE_SERVER_IO_ERROR", "1")
+        .assert()
+        .success()
+        .stdout(predicate::str::starts_with("rustc "))
+        .stderr(predicate::str::contains("compiling locally instead"));
+
+    Ok(())
+}
+
+#[cfg(unix)]
+#[test]
+#[serial]
+fn test_server_start_failure_errors() -> Result<()> {
+    let (_tempdir, mut command) = command_with_server_start_failure()?;
+
+    command
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains("sccache: error"));
+
+    Ok(())
+}
+
 #[test]
 #[serial]
 #[cfg(feature = "gcs")]