feat(vmm): preserve serial logs across VM restarts

QEMU truncates serial.log on each boot, losing previous crash/panic
logs that are critical for debugging restart loops.

- Append serial.log to serial.history.log with boot timestamp separator
  before each VM restart, capped at a configurable max size (default 4MB)
- Add boot separator timestamps to stdout.log and stderr.log so
  individual boot sessions are clearly delimited
- Add `serial_history_max_bytes` config option (supports human-readable
  sizes like "4MB", "512KB")

Author: kvinwang

vmm/src/app.rs

@@ -242,6 +242,11 @@ impl App {
                     fs::remove_file(path)?;
                 }
             }
+            // Append current serial.log to serial.history.log before QEMU truncates it.
+            rotate_serial_log(&work_dir, self.config.cvm.serial_history_max_bytes);
+            // Add boot separator to stdout/stderr (they are opened in append mode).
+            append_boot_separator(&work_dir.stdout_file());
+            append_boot_separator(&work_dir.stderr_file());
 
             let devices = self.try_allocate_gpus(&vm_config.manifest)?;
             let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?;
@@ -1051,6 +1056,64 @@ impl App {
     }
 }
 
+/// Append a boot separator line with timestamp to an append-mode log file.
+fn append_boot_separator(path: &std::path::Path) {
+    use std::io::Write;
+    if !path.exists() {
+        return;
+    }
+    let Ok(mut file) = std::fs::OpenOptions::new().append(true).open(path) else {
+        return;
+    };
+    let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now());
+    let _ = writeln!(file, "\n===== boot @ {timestamp} =====\n");
+}
+
+/// Append current serial.log into serial.history.log with a boot separator,
+/// then truncate history if it exceeds `max_bytes`.
+fn rotate_serial_log(work_dir: &VmWorkDir, max_bytes: u64) {
+    use std::io::Write;
+
+    let serial = work_dir.serial_file();
+    if !serial.exists() {
+        return;
+    }
+    let Ok(content) = fs::read(&serial) else {
+        return;
+    };
+    if content.is_empty() {
+        return;
+    }
+    let history = work_dir.serial_history_file();
+    let Ok(mut file) = std::fs::OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(&history)
+    else {
+        return;
+    };
+    let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now());
+    let _ = writeln!(file, "\n===== boot @ {timestamp} =====\n");
+    let _ = file.write_all(&content);
+    drop(file);
+
+    // Truncate from the front if history exceeds max_bytes.
+    if let Ok(meta) = fs::metadata(&history) {
+        if meta.len() > max_bytes {
+            if let Ok(data) = fs::read(&history) {
+                let skip = data.len() - max_bytes as usize;
+                // Find the next newline after skip point to avoid cutting mid-line.
+                let start = data[skip..]
+                    .iter()
+                    .position(|&b| b == b'\n')
+                    .map(|p| skip + p + 1)
+                    .unwrap_or(skip);
+                let _ = fs::write(&history, &data[start..]);
+            }
+        }
+    }
+}
+
 pub(crate) fn make_sys_config(cfg: &Config, manifest: &Manifest) -> Result<String> {
     let image_path = cfg.image_path.join(&manifest.image);
     let image = Image::load(image_path).context("Failed to load image info")?;

vmm/src/app/qemu.rs

@@ -1042,6 +1042,10 @@ impl VmWorkDir {
         self.workdir.join("serial.log")
     }
 
+    pub fn serial_history_file(&self) -> PathBuf {
+        self.workdir.join("serial.history.log")
+    }
+
     pub fn serial_pty(&self) -> PathBuf {
         self.workdir.join("serial.pty")
     }

vmm/src/config.rs

@@ -203,6 +203,13 @@ pub struct CvmConfig {
     /// SMBIOS product information for cloud environment detection
     #[serde(default)]
     pub product: ProductConfig,
+
+    /// Max size in bytes for serial.history.log (default 4MB).
+    /// Previous boot serial logs are appended here before each restart.
+    /// Accepts human-readable sizes like "4MB", "512KB".
+    #[serde(default = "default_serial_history_max_bytes")]
+    #[serde(with = "size_parser::human_size")]
+    pub serial_history_max_bytes: u64,
 }
 
 /// SMBIOS product information configuration.
@@ -452,6 +459,10 @@ pub struct KeyProviderConfig {
     pub port: u16,
 }
 
+fn default_serial_history_max_bytes() -> u64 {
+    4 * 1024 * 1024 // 4MB
+}
+
 const CLIENT_CONF_PATH: &str = "/etc/dstack/client.conf";
 fn read_qemu_path_from_client_conf() -> Option<PathBuf> {
     #[derive(Debug, Deserialize)]