diff --git a/crates/integration-tests/src/tests/libvirt_port_forward.rs b/crates/integration-tests/src/tests/libvirt_port_forward.rs index 125bffea3..657d7d1f9 100644 --- a/crates/integration-tests/src/tests/libvirt_port_forward.rs +++ b/crates/integration-tests/src/tests/libvirt_port_forward.rs @@ -272,6 +272,7 @@ fn test_libvirt_port_forward_connectivity() -> Result<()> { &format!("{}:8080", host_port), "--filesystem", "ext4", + "--ssh-wait", &test_image, ]) .expect("Failed to run libvirt run with port forwarding"); @@ -289,12 +290,8 @@ fn test_libvirt_port_forward_connectivity() -> Result<()> { println!("Successfully created domain: {}", domain_name); - // Wait for VM to boot and SSH to become available - println!("Waiting for VM to boot and SSH to become available..."); - if let Err(e) = wait_for_ssh_available(&domain_name, 180) { - cleanup_domain(&domain_name); - panic!("Failed to establish SSH connection: {}", e); - } + // SSH is already available due to --ssh-wait flag + println!("✓ SSH is ready (via --ssh-wait)"); // Start a simple HTTP server on port 8080 inside the VM using Python println!("Starting HTTP server on port 8080 inside VM..."); @@ -445,56 +442,6 @@ fn cleanup_domain(domain_name: &str) { } } -/// Wait for SSH to become available on a domain with a timeout -fn wait_for_ssh_available( - domain_name: &str, - timeout_secs: u64, -) -> Result<(), Box> { - let start_time = std::time::Instant::now(); - let timeout_duration = std::time::Duration::from_secs(timeout_secs); - - println!( - "Waiting for SSH to become available on domain: {}", - domain_name - ); - - loop { - // Check if we've exceeded the timeout before attempting SSH - if start_time.elapsed() >= timeout_duration { - return Err(format!("Timeout waiting for SSH after {} seconds", timeout_secs).into()); - } - - // Try a simple SSH command to test connectivity with a short timeout (5 seconds) - // This prevents each SSH attempt from hanging for the default 30 seconds - let ssh_test = run_bcvk(&[ - "libvirt", - "ssh", - "--timeout", - "5", - domain_name, - "--", - "echo", - "ssh-ready", - ]); - - match ssh_test { - Ok(output) if output.success() => { - println!("✓ SSH is now available"); - return Ok(()); - } - Ok(_) => { - // SSH command failed, but that's expected while VM is booting - } - Err(e) => { - println!("SSH test error (expected while booting): {}", e); - } - } - - // Wait 2 seconds before next attempt (since we already waited 5 seconds for SSH timeout) - std::thread::sleep(std::time::Duration::from_secs(2)); - } -} - /// Find an available port on the host fn find_available_port() -> Result { use std::net::TcpListener; diff --git a/crates/integration-tests/src/tests/libvirt_verb.rs b/crates/integration-tests/src/tests/libvirt_verb.rs index 0fa43cea7..f306daca7 100644 --- a/crates/integration-tests/src/tests/libvirt_verb.rs +++ b/crates/integration-tests/src/tests/libvirt_verb.rs @@ -486,46 +486,6 @@ fn check_libvirt_supports_readonly_virtiofs() -> Result { Ok(supports_readonly) } -/// Wait for SSH to become available on a domain with a timeout -fn wait_for_ssh_available( - domain_name: &str, - timeout_secs: u64, -) -> Result<(), Box> { - let start_time = std::time::Instant::now(); - let timeout_duration = std::time::Duration::from_secs(timeout_secs); - - println!( - "Waiting for SSH to become available on domain: {}", - domain_name - ); - - loop { - // Try a simple SSH command to test connectivity - let ssh_test = run_bcvk(&["libvirt", "ssh", domain_name, "--", "echo", "ssh-ready"]); - - match ssh_test { - Ok(output) if output.success() => { - println!("✓ SSH is now available"); - return Ok(()); - } - Ok(_) => { - // SSH command failed, but that's expected while VM is booting - } - Err(e) => { - println!("SSH test error (expected while booting): {}", e); - } - } - - // Check if we've exceeded the timeout - if start_time.elapsed() >= timeout_duration { - return Err(format!("Timeout waiting for SSH after {} seconds", timeout_secs).into()); - } - - // Wait 5 seconds before next attempt - std::thread::sleep(std::time::Duration::from_secs(5)); - } -} - /// Test VM startup and shutdown with libvirt run fn test_libvirt_run_vm_lifecycle() -> Result<()> { let bck = get_bck_command()?; @@ -664,6 +624,7 @@ fn test_libvirt_run_bind_storage_ro() -> Result<()> { "--bind-storage-ro", "--filesystem", "ext4", + "--ssh-wait", &test_image, ]) .expect("Failed to run libvirt run with --bind-storage-ro"); @@ -730,13 +691,10 @@ fn test_libvirt_run_bind_storage_ro() -> Result<()> { println!("✓ Container storage mount is configured as read-only"); println!("✓ hoststorage tag is present in filesystem configuration"); - // Wait for VM to boot and SSH to become available - if let Err(e) = wait_for_ssh_available(&domain_name, 180) { - cleanup_domain(&domain_name); - panic!("Failed to establish SSH connection: {}", e); - } + // SSH is already available due to --ssh-wait flag + println!("✓ SSH is ready (via --ssh-wait)"); - // Wait for VM to boot and automatic mount to complete + // Wait for automatic mount to complete println!("Waiting for VM to boot and automatic mount to complete..."); std::thread::sleep(std::time::Duration::from_secs(10)); diff --git a/crates/kit/src/libvirt/run.rs b/crates/kit/src/libvirt/run.rs index f777a5ecd..3f878f2fa 100644 --- a/crates/kit/src/libvirt/run.rs +++ b/crates/kit/src/libvirt/run.rs @@ -19,6 +19,9 @@ use crate::libvirt::domain::VirtiofsFilesystem; use crate::utils::parse_memory_to_mb; use crate::xml_utils; +/// SSH wait timeout in seconds +const SSH_WAIT_TIMEOUT_SECONDS: u64 = 180; + /// Create a virsh command with optional connection URI pub(super) fn virsh_command(connect_uri: Option<&str>) -> Result { let mut cmd = std::process::Command::new("virsh"); @@ -253,6 +256,10 @@ pub struct LibvirtRunOpts { #[clap(long)] pub ssh: bool, + /// Wait for SSH to become available and verify connectivity (for testing) + #[clap(long, conflicts_with = "ssh")] + pub ssh_wait: bool, + /// Mount host container storage (RO) at /run/host-container-storage #[clap(long = "bind-storage-ro")] pub bind_storage_ro: bool, @@ -319,6 +326,60 @@ impl LibvirtRunOpts { } } +/// Wait for SSH to become available on a libvirt domain +/// +/// Polls SSH connectivity by attempting simple commands until successful or timeout. +fn wait_for_ssh_ready( + global_opts: &crate::libvirt::LibvirtOptions, + domain_name: &str, + timeout_secs: u64, +) -> Result<()> { + use std::time::Duration; + + debug!( + "Waiting for SSH to become available on domain '{}' (timeout: {}s)", + domain_name, timeout_secs + ); + + // Create progress bar + let pb = crate::boot_progress::create_boot_progress_bar(); + pb.set_message("Waiting for SSH to become available..."); + + // Clone values for closure + let global_opts_clone = global_opts.clone(); + let domain_name_clone = domain_name.to_string(); + + // Use shared polling function with libvirt-specific test + let (_elapsed, pb) = crate::utils::wait_for_readiness( + pb, + "Waiting for SSH", + || { + // Create a test SSH connection with short timeout + let ssh_opts = crate::libvirt::ssh::LibvirtSshOpts { + domain_name: domain_name_clone.clone(), + user: "root".to_string(), + command: vec!["true".to_string()], // Simple command to test connectivity + strict_host_keys: false, + timeout: 5, // Short timeout for each attempt + log_level: "ERROR".to_string(), + extra_options: vec![], + suppress_output: true, // Suppress error messages during connectivity testing + }; + + // Try to connect + match crate::libvirt::ssh::run_ssh_impl(&global_opts_clone, ssh_opts) { + Ok(_) => Ok(true), + Err(_) => Ok(false), + } + }, + Duration::from_secs(timeout_secs), + Duration::from_secs(2), // Poll every 2 seconds + )?; + + pb.finish_and_clear(); + Ok(()) +} + /// Execute the libvirt run command pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtRunOpts) -> Result<()> { use crate::images; @@ -445,12 +506,21 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtRunOpts) - } } - if opts.ssh { + if opts.ssh_wait { + // Wait for SSH to be ready and verify connectivity + wait_for_ssh_ready(global_opts, &vm_name, SSH_WAIT_TIMEOUT_SECONDS)?; + println!("Ready; use bcvk libvirt ssh to connect"); + Ok(()) + } else if opts.ssh { + // Wait for SSH then enter interactive shell + wait_for_ssh_ready(global_opts, &vm_name, SSH_WAIT_TIMEOUT_SECONDS)?; + // Use the libvirt SSH functionality directly let ssh_opts = crate::libvirt::ssh::LibvirtSshOpts { domain_name: vm_name, user: "root".to_string(), command: vec![], + suppress_output: false, strict_host_keys: false, timeout: 30, log_level: "ERROR".to_string(), diff --git a/crates/kit/src/libvirt/ssh.rs b/crates/kit/src/libvirt/ssh.rs index 2ffce1882..577028cdf 100644 --- a/crates/kit/src/libvirt/ssh.rs +++ b/crates/kit/src/libvirt/ssh.rs @@ -46,6 +46,10 @@ pub struct LibvirtSshOpts { /// Extra SSH options in key=value format #[clap(long)] pub extra_options: Vec, + + /// Suppress stdout/stderr output (for connectivity testing) + #[clap(skip)] + pub suppress_output: bool, } /// SSH configuration extracted from domain metadata @@ -316,13 +320,17 @@ impl LibvirtSshOpts { .map_err(|e| eyre!("Failed to execute SSH command: {}", e))?; if !output.stdout.is_empty() { - // Forward stdout to parent process - print!("{}", String::from_utf8_lossy(&output.stdout)); + if !self.suppress_output { + // Forward stdout to parent process + print!("{}", String::from_utf8_lossy(&output.stdout)); + } debug!("SSH stdout: {}", String::from_utf8_lossy(&output.stdout)); } if !output.stderr.is_empty() { - // Forward stderr to parent process - eprint!("{}", String::from_utf8_lossy(&output.stderr)); + if !self.suppress_output { + // Forward stderr to parent process + eprint!("{}", String::from_utf8_lossy(&output.stderr)); + } debug!("SSH stderr: {}", String::from_utf8_lossy(&output.stderr)); } diff --git a/crates/kit/src/libvirt/start.rs b/crates/kit/src/libvirt/start.rs index f1d4c5568..729188dd7 100644 --- a/crates/kit/src/libvirt/start.rs +++ b/crates/kit/src/libvirt/start.rs @@ -45,6 +45,7 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtStartOpts) timeout: 30, log_level: "ERROR".to_string(), extra_options: vec![], + suppress_output: false, }; return crate::libvirt::ssh::run(global_opts, ssh_opts); } @@ -81,6 +82,7 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtStartOpts) timeout: 30, log_level: "ERROR".to_string(), extra_options: vec![], + suppress_output: false, }; crate::libvirt::ssh::run(global_opts, ssh_opts) } else { diff --git a/crates/kit/src/run_ephemeral_ssh.rs b/crates/kit/src/run_ephemeral_ssh.rs index 32b822102..6260a3190 100644 --- a/crates/kit/src/run_ephemeral_ssh.rs +++ b/crates/kit/src/run_ephemeral_ssh.rs @@ -3,8 +3,7 @@ use color_eyre::Result; use indicatif::ProgressBar; use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use tracing::debug; use crate::run_ephemeral::{run_detached, RunEphemeralOpts}; @@ -116,38 +115,32 @@ pub fn wait_for_ssh_ready( let timeout = timeout.unwrap_or(SSH_TIMEOUT); let (_, progress) = wait_for_vm_ssh(container_name, Some(timeout), progress)?; - debug!("Polling SSH connectivity...",); - let start_time = Instant::now(); + debug!("Polling SSH connectivity..."); // Use SSH options optimized for connectivity testing let ssh_options = crate::ssh::SshConnectionOptions::for_connectivity_test(); - - let mut attempt = 0; - while start_time.elapsed() < timeout { - attempt += 1; - progress.set_message(format!("Polling for SSH readiness (attempt {attempt})")); - - // Try to connect via SSH and run a simple command using the centralized SSH function - let status = crate::ssh::connect( - container_name, - vec!["true".to_string()], // Just run 'true' to test connectivity - &ssh_options, - ); - - if let Ok(exit_status) = status { - if exit_status.success() { - debug!("SSH connection successful, VM is ready"); - return Ok((start_time.elapsed(), progress)); + let container_name = container_name.to_string(); + + // Use shared polling function with container-specific test + crate::utils::wait_for_readiness( + progress, + "Waiting for SSH", + || { + // Try to connect via SSH and run a simple command + let status = crate::ssh::connect( + &container_name, + vec!["true".to_string()], // Just run 'true' to test connectivity + &ssh_options, + ); + + match status { + Ok(exit_status) if exit_status.success() => Ok(true), + _ => Ok(false), } - } - - thread::sleep(Duration::from_secs(1)); - } - - Err(color_eyre::eyre::eyre!( - "Timeout waiting for SSH connectivity after {}s", - timeout.as_secs() - )) + }, + timeout, + Duration::from_secs(1), // Poll every 1 second + ) } /// Run an ephemeral pod and immediately SSH into it, with lifecycle binding diff --git a/crates/kit/src/utils.rs b/crates/kit/src/utils.rs index b6b08c498..ac0649184 100644 --- a/crates/kit/src/utils.rs +++ b/crates/kit/src/utils.rs @@ -1,10 +1,79 @@ use camino::{Utf8Path, Utf8PathBuf}; +use color_eyre::eyre::{eyre, Context}; +use color_eyre::Result; +use indicatif::ProgressBar; use std::io::{Seek as _, Write as _}; use std::os::fd::OwnedFd; +use std::time::{Duration, Instant}; use cap_std_ext::cap_std::io_lifetimes::AsFilelike as _; -use color_eyre::eyre::{eyre, Context}; -use color_eyre::Result; +use tracing::debug; + +/// Wait for a condition to become ready with progress indication +/// +/// Generic polling function that repeatedly tests a condition until it succeeds or +/// times out. Updates a progress bar with attempt count and elapsed time. +/// +/// # Arguments +/// +/// * `progress` - Progress bar to update with attempt status +/// * `message` - Message to display (e.g., "Waiting for SSH") +/// * `test_fn` - Function that tests the readiness condition, returns Ok(true) on success +/// * `timeout` - Maximum duration to wait +/// * `poll_interval` - Duration to wait between test attempts +/// +/// # Returns +/// +/// Returns the elapsed duration and progress bar on success, or an error on timeout +pub fn wait_for_readiness( + progress: ProgressBar, + message: &str, + mut test_fn: F, + timeout: Duration, + poll_interval: Duration, +) -> Result<(Duration, ProgressBar)> +where + F: FnMut() -> Result, +{ + let start_time = Instant::now(); + + debug!("Polling for readiness (timeout: {}s)", timeout.as_secs()); + + let mut attempt = 0; + while start_time.elapsed() < timeout { + attempt += 1; + + progress.set_message(format!( + "{} (attempt {}, elapsed: {}s)", + message, + attempt, + start_time.elapsed().as_secs() + )); + + // Try to connect + match test_fn() { + Ok(true) => { + debug!("Readiness check successful after {} attempts", attempt); + return Ok((start_time.elapsed(), progress)); + } + Ok(false) => { + debug!("Readiness check attempt {} returned false", attempt); + } + Err(e) => { + debug!("Readiness check attempt {} failed: {}", attempt, e); + } + } + + // Wait before next attempt + std::thread::sleep(poll_interval); + } + + Err(eyre!( + "Timeout waiting for readiness after {}s ({} attempts)", + timeout.as_secs(), + attempt + )) +} /// Creates a sealed memory file descriptor for secure data transfer. /// The sealed memfd cannot be modified after creation, providing tamper protection. diff --git a/docs/src/man/bcvk-libvirt-run.md b/docs/src/man/bcvk-libvirt-run.md index aebb25921..41e36f0b5 100644 --- a/docs/src/man/bcvk-libvirt-run.md +++ b/docs/src/man/bcvk-libvirt-run.md @@ -95,6 +95,10 @@ Run a bootable container as a persistent VM Automatically SSH into the VM after creation +**--ssh-wait** + + Wait for SSH to become available and verify connectivity (for testing) + **--bind-storage-ro** Mount host container storage (RO) at /run/host-container-storage