feat(box): tag container logs stdout vs stderr (split virtio-console)

container.json hard-coded stream="stdout" for every line — a container's
stderr (`echo x 1>&2`) was mislabeled stdout, the last Docker json-file
divergence. (The pipe-capture approach to separate streams was adversarially
rejected — it can freeze the container / hang shutdown.)

Use libkrun's 3-fd virtio-console instead: guest stdout -> console.log, stderr
-> console.err.log (krun_add_virtio_console_default, after
krun_disable_implicit_console). EMPIRICALLY verified libkrun separates the
streams with NO guest change and no pipe (console devices aren't bounded pipes
the container can block on, so no freeze/deadlock).

- shim: split console by default (BOX_NO_SPLIT_STDERR forces legacy merged;
  falls back to merged if the err file can't be opened).
- core/log.rs: the processor tails BOTH files on scoped threads sharing one
  RotatingWriter, tagging each line's stream; filters libkrun's `init.krun:`
  preamble on BOTH streams (it can land on either).
- cli: foreground `run` and `attach` tail console.err.log to the terminal's
  stderr (stdout->fd1, stderr->fd2, like Docker); `logs` routes stderr lines to
  its stderr.

KVM-verified: `echo OUT; echo ERR 1>&2` -> container.json tags OUT=stdout,
ERR=stderr, zero init.krun noise; foreground shows stdout on fd1 + stderr on
fd2; detached logs still complete; exit code 5 propagates. Completes Docker
json-file log parity.

Author: Roy Lin

src/cli/src/commands/attach.rs

@@ -53,8 +53,12 @@ pub async fn execute(args: AttachArgs) -> Result<(), Box<dyn std::error::Error>>
 
     println!("Attached to box {}. Press Ctrl-C to detach.", record.name);
 
+    let console_err = console_log.with_file_name("console.err.log");
     let log_handle = tokio::spawn(async move {
-        super::tail_file(&console_log).await;
+        tokio::join!(
+            super::tail_file(&console_log),
+            super::tail_file_stream(&console_err, true),
+        );
     });
 
     let _ = tokio::signal::ctrl_c().await;

src/cli/src/commands/logs.rs

@@ -241,11 +241,18 @@ fn print_json_line(
         }
     }
 
-    // The log field already contains the trailing newline
-    if timestamps {
-        print!("{} {}", entry.time, entry.log);
+    // The log field already contains the trailing newline. Route stderr lines to
+    // the terminal's stderr, like Docker `logs`.
+    let line = if timestamps {
+        format!("{} {}", entry.time, entry.log)
+    } else {
+        entry.log.clone()
+    };
+    if entry.stream == "stderr" {
+        use std::io::Write as _;
+        let _ = std::io::stderr().write_all(line.as_bytes());
     } else {
-        print!("{}", entry.log);
+        print!("{line}");
     }
 }
 

src/cli/src/commands/mod.rs

@@ -241,6 +241,14 @@ pub(crate) fn resolve_box_rootfs(box_dir: &std::path::Path) -> Option<PathBuf> {
 /// Waits for the file to exist, then continuously reads and prints new data.
 /// Used by `run` (foreground mode) and `attach`.
 pub(crate) async fn tail_file(path: &std::path::Path) {
+    tail_file_stream(path, false).await;
+}
+
+/// Tail a console file to the terminal. `to_stderr` routes it to the terminal's
+/// stderr — used for the container's stderr console (console.err.log) so a
+/// foreground `run`/`attach` shows stdout and stderr like Docker.
+pub(crate) async fn tail_file_stream(path: &std::path::Path, to_stderr: bool) {
+    use std::io::Write as _;
     use tokio::io::AsyncReadExt;
 
     // Wait for file to exist
@@ -264,7 +272,13 @@ pub(crate) async fn tail_file(path: &std::path::Path) {
             }
             Ok(n) => {
                 let text = String::from_utf8_lossy(&buf[..n]);
-                print!("{text}");
+                if to_stderr {
+                    let mut err = std::io::stderr();
+                    let _ = err.write_all(text.as_bytes());
+                    let _ = err.flush();
+                } else {
+                    print!("{text}");
+                }
             }
             Err(_) => break,
         }

src/cli/src/commands/run.rs

@@ -624,8 +624,14 @@ async fn run_foreground(
     );
 
     let console_log = ctx.box_dir.join("logs").join("console.log");
+    let console_err = ctx.box_dir.join("logs").join("console.err.log");
     let log_handle = tokio::spawn(async move {
-        super::tail_file(&console_log).await;
+        // Stream the container's stdout (console.log) and stderr
+        // (console.err.log, split console) to the terminal's stdout/stderr.
+        tokio::join!(
+            super::tail_file(&console_log),
+            super::tail_file_stream(&console_err, true),
+        );
     });
 
     let name = ctx.name.clone();

src/core/src/log.rs

@@ -242,41 +242,75 @@ fn open_console(console_log: &Path, stop: &AtomicBool) -> Option<std::fs::File>
 }
 
 /// Tail console.log and write one Docker-style JSON record per container line.
+/// The stderr companion to `console.log` (libkrun's 3-fd console sends guest
+/// stderr here, stdout to `console.log`).
+pub fn stderr_console_path(console_log: &Path) -> PathBuf {
+    console_log.with_file_name("console.err.log")
+}
+
+/// Tail one console file, emitting each container line via `emit(line, stream)`.
+/// `filter_noise` drops libkrun's `init.krun:` preamble (only on the stdout
+/// console). Blocks until `stop` is set and the file is drained.
+fn run_tagged_tail(
+    file: &Path,
+    stream: &str,
+    filter_noise: bool,
+    stop: &AtomicBool,
+    emit: &(dyn Fn(&str, &str) + Sync),
+) {
+    let f = match open_console(file, stop) {
+        Some(f) => f,
+        None => return,
+    };
+    let mut reader = BufReader::new(f);
+    let mut buf = String::new();
+    while let Some(line) = tail_next_line(&mut reader, &mut buf, stop) {
+        if filter_noise && is_runtime_console_noise(&line) {
+            continue;
+        }
+        emit(&line, stream);
+    }
+}
+
 fn run_json_file_processor(
     console_log: &Path,
     log_dir: &Path,
     max_size: u64,
     max_file: u32,
     stop: &AtomicBool,
 ) {
-    let file = match open_console(console_log, stop) {
-        Some(f) => f,
-        None => return,
-    };
-    let mut reader = BufReader::new(file);
     let json_path = json_log_path(log_dir);
-    let mut writer = match RotatingWriter::new(&json_path, max_size, max_file) {
+    let writer = std::sync::Mutex::new(match RotatingWriter::new(&json_path, max_size, max_file) {
         Ok(w) => w,
         Err(_) => return,
-    };
+    });
+    let err_log = stderr_console_path(console_log);
 
-    let mut buf = String::new();
-    while let Some(line) = tail_next_line(&mut reader, &mut buf, stop) {
-        if is_runtime_console_noise(&line) {
-            continue;
-        }
+    // Write one tagged JSON record per line; shared by the stdout and stderr
+    // tail threads (the Mutex serializes their interleave into container.json).
+    let emit = |line: &str, stream: &str| {
         let entry = LogEntry {
             log: format!("{line}\n"),
-            stream: "stdout".to_string(),
+            stream: stream.to_string(),
             time: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true),
         };
         if let Ok(json) = serde_json::to_string(&entry) {
-            let _ = writer.write_line(&json);
+            if let Ok(mut w) = writer.lock() {
+                let _ = w.write_line(&json);
+            }
         }
-    }
+    };
+    let emit: &(dyn Fn(&str, &str) + Sync) = &emit;
+
+    std::thread::scope(|s| {
+        s.spawn(|| run_tagged_tail(console_log, "stdout", true, stop, emit));
+        // libkrun's `init.krun:` preamble can land on EITHER stream, so filter
+        // the noise on stderr too.
+        s.spawn(|| run_tagged_tail(&err_log, "stderr", true, stop, emit));
+    });
 }
 
-/// Forward console.log lines to a syslog endpoint via UDP or TCP.
+/// Forward both console streams (stdout + stderr) to a syslog endpoint.
 fn run_syslog_processor(
     console_log: &Path,
     address: &str,
@@ -286,54 +320,53 @@ fn run_syslog_processor(
 ) {
     use std::net::UdpSocket;
 
-    let file = match open_console(console_log, stop) {
-        Some(f) => f,
-        None => return,
-    };
     let (proto, addr) = if let Some(rest) = address.strip_prefix("udp://") {
         ("udp", rest)
     } else if let Some(rest) = address.strip_prefix("tcp://") {
         ("tcp", rest)
     } else {
         ("udp", address)
     };
-
-    let mut reader = BufReader::new(file);
-    let mut buf = String::new();
+    let err_log = stderr_console_path(console_log);
 
     match proto {
         "udp" => {
             let socket = match UdpSocket::bind("0.0.0.0:0") {
                 Ok(s) => s,
                 Err(_) => return,
             };
-            while let Some(line) = tail_next_line(&mut reader, &mut buf, stop) {
-                if is_runtime_console_noise(&line) {
-                    continue;
-                }
-                // RFC 3164: <priority>tag: message; daemon(3)*8 + info(6) = 30.
+            // RFC 3164: <priority>tag: message; daemon(3)*8 + info(6) = 30.
+            let emit = |line: &str, _stream: &str| {
                 let msg = format!("<30>{tag}: {line}");
                 let _ = socket.send_to(msg.as_bytes(), addr);
-            }
+            };
+            let emit: &(dyn Fn(&str, &str) + Sync) = &emit;
+            std::thread::scope(|s| {
+                s.spawn(|| run_tagged_tail(console_log, "stdout", true, stop, emit));
+                s.spawn(|| run_tagged_tail(&err_log, "stderr", true, stop, emit));
+            });
         }
         "tcp" => {
-            let mut stream = match std::net::TcpStream::connect(addr) {
-                Ok(s) => s,
+            let stream = match std::net::TcpStream::connect(addr) {
+                Ok(s) => std::sync::Mutex::new(s),
                 Err(_) => return,
             };
-            while let Some(line) = tail_next_line(&mut reader, &mut buf, stop) {
-                if is_runtime_console_noise(&line) {
-                    continue;
-                }
+            let emit = |line: &str, _stream: &str| {
                 let msg = format!("<30>{tag}: {line}\n");
-                if stream.write_all(msg.as_bytes()).is_err() {
-                    stream = match std::net::TcpStream::connect(addr) {
-                        Ok(s) => s,
-                        Err(_) => return,
-                    };
-                    let _ = stream.write_all(msg.as_bytes());
+                if let Ok(mut s) = stream.lock() {
+                    if s.write_all(msg.as_bytes()).is_err() {
+                        if let Ok(news) = std::net::TcpStream::connect(addr) {
+                            *s = news;
+                            let _ = s.write_all(msg.as_bytes());
+                        }
+                    }
                 }
-            }
+            };
+            let emit: &(dyn Fn(&str, &str) + Sync) = &emit;
+            std::thread::scope(|sc| {
+                sc.spawn(|| run_tagged_tail(console_log, "stdout", true, stop, emit));
+                sc.spawn(|| run_tagged_tail(&err_log, "stderr", true, stop, emit));
+            });
         }
         _ => {}
     }

src/shim/src/krun/context.rs

@@ -21,7 +21,8 @@ use libkrun_sys::{krun_add_net_tcp, krun_add_vsock_port_windows, krun_set_kernel
 #[cfg(target_os = "linux")]
 use libkrun_sys::{krun_add_net_unixstream, krun_split_irqchip};
 use libkrun_sys::{
-    krun_add_virtiofs, krun_create_ctx, krun_free_ctx, krun_init_log, krun_set_console_output,
+    krun_add_virtio_console_default, krun_add_virtiofs, krun_create_ctx,
+    krun_disable_implicit_console, krun_free_ctx, krun_init_log, krun_set_console_output,
     krun_set_env, krun_set_exec, krun_set_rlimits, krun_set_root, krun_set_vm_config,
     krun_set_workdir, krun_setgid, krun_setuid, krun_start_enter,
 };
@@ -437,6 +438,25 @@ impl KrunContext {
         )
     }
 
+    /// Replace the implicit console with a virtio-console whose stdout and
+    /// stderr go to SEPARATE host fds (so the guest's stderr can be tagged).
+    /// `input_fd` may be -1 (no stdin). Caller owns the fds for the VM lifetime.
+    pub unsafe fn add_split_console(
+        &self,
+        input_fd: i32,
+        output_fd: i32,
+        err_fd: i32,
+    ) -> Result<()> {
+        check_status(
+            "krun_disable_implicit_console",
+            krun_disable_implicit_console(self.ctx_id),
+        )?;
+        check_status(
+            "krun_add_virtio_console_default",
+            krun_add_virtio_console_default(self.ctx_id, input_fd, output_fd, err_fd),
+        )
+    }
+
     /// Enable split IRQ chip mode (required for TEE VMs).
     ///
     /// This must be called before starting a TEE-enabled VM.

src/shim/src/main.rs

@@ -756,8 +756,37 @@ unsafe fn configure_and_start_vm(spec: &InstanceSpec) -> Result<()> {
                 message: format!("Invalid console output path: {}", console_path.display()),
                 hint: None,
             })?;
-        tracing::debug!(console_path = console_str, "Redirecting console output");
-        ctx.set_console_output(console_str)?;
+        // Split console: guest stdout -> console.log, stderr -> console.err.log
+        // (libkrun's 3-fd virtio-console separates the streams), so the log
+        // processor can tag each line's stream like Docker's json-file driver.
+        // Falls back to the merged single-file console if the err file can't be
+        // opened. BOX_NO_SPLIT_STDERR forces the legacy merged behavior.
+        use std::os::unix::io::AsRawFd;
+        let opened = if std::env::var_os("BOX_NO_SPLIT_STDERR").is_none() {
+            let err_path = console_path.with_file_name("console.err.log");
+            let open = |p: &std::path::Path| {
+                std::fs::OpenOptions::new().create(true).append(true).open(p)
+            };
+            match (open(console_path), open(&err_path)) {
+                (Ok(out_f), Ok(err_f)) => Some((out_f, err_f)),
+                _ => None,
+            }
+        } else {
+            None
+        };
+        match opened {
+            Some((out_f, err_f)) => {
+                ctx.add_split_console(-1, out_f.as_raw_fd(), err_f.as_raw_fd())?;
+                // Keep the fds open for the VM's lifetime.
+                std::mem::forget(out_f);
+                std::mem::forget(err_f);
+                tracing::debug!("split console enabled (stdout/stderr separated)");
+            }
+            None => {
+                tracing::debug!(console_path = console_str, "Redirecting console output (merged)");
+                ctx.set_console_output(console_str)?;
+            }
+        }
     }
 
     // Configure TEE if specified (only available on Linux with SEV support)