Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 3 additions & 56 deletions crates/integration-tests/src/tests/libvirt_port_forward.rs
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ fn test_libvirt_port_forward_connectivity() -> Result<()> {
&format!("{}:8080", host_port),
"--filesystem",
"ext4",
"--ssh-wait",
&test_image,
])
.expect("Failed to run libvirt run with port forwarding");
Expand All @@ -289,12 +290,8 @@ fn test_libvirt_port_forward_connectivity() -> Result<()> {

println!("Successfully created domain: {}", domain_name);

// Wait for VM to boot and SSH to become available
println!("Waiting for VM to boot and SSH to become available...");
if let Err(e) = wait_for_ssh_available(&domain_name, 180) {
cleanup_domain(&domain_name);
panic!("Failed to establish SSH connection: {}", e);
}
// SSH is already available due to --ssh-wait flag
println!("✓ SSH is ready (via --ssh-wait)");

// Start a simple HTTP server on port 8080 inside the VM using Python
println!("Starting HTTP server on port 8080 inside VM...");
Expand Down Expand Up @@ -445,56 +442,6 @@ fn cleanup_domain(domain_name: &str) {
}
}

/// Wait for SSH to become available on a domain with a timeout
fn wait_for_ssh_available(
domain_name: &str,
timeout_secs: u64,
) -> Result<(), Box<dyn std::error::Error>> {
let start_time = std::time::Instant::now();
let timeout_duration = std::time::Duration::from_secs(timeout_secs);

println!(
"Waiting for SSH to become available on domain: {}",
domain_name
);

loop {
// Check if we've exceeded the timeout before attempting SSH
if start_time.elapsed() >= timeout_duration {
return Err(format!("Timeout waiting for SSH after {} seconds", timeout_secs).into());
}

// Try a simple SSH command to test connectivity with a short timeout (5 seconds)
// This prevents each SSH attempt from hanging for the default 30 seconds
let ssh_test = run_bcvk(&[
"libvirt",
"ssh",
"--timeout",
"5",
domain_name,
"--",
"echo",
"ssh-ready",
]);

match ssh_test {
Ok(output) if output.success() => {
println!("✓ SSH is now available");
return Ok(());
}
Ok(_) => {
// SSH command failed, but that's expected while VM is booting
}
Err(e) => {
println!("SSH test error (expected while booting): {}", e);
}
}

// Wait 2 seconds before next attempt (since we already waited 5 seconds for SSH timeout)
std::thread::sleep(std::time::Duration::from_secs(2));
}
}

/// Find an available port on the host
fn find_available_port() -> Result<u16> {
use std::net::TcpListener;
Expand Down
50 changes: 4 additions & 46 deletions crates/integration-tests/src/tests/libvirt_verb.rs
Original file line number Diff line number Diff line change
Expand Up @@ -486,46 +486,6 @@ fn check_libvirt_supports_readonly_virtiofs() -> Result<bool> {
Ok(supports_readonly)
}

/// Wait for SSH to become available on a domain with a timeout
fn wait_for_ssh_available(
domain_name: &str,
timeout_secs: u64,
) -> Result<(), Box<dyn std::error::Error>> {
let start_time = std::time::Instant::now();
let timeout_duration = std::time::Duration::from_secs(timeout_secs);

println!(
"Waiting for SSH to become available on domain: {}",
domain_name
);

loop {
// Try a simple SSH command to test connectivity
let ssh_test = run_bcvk(&["libvirt", "ssh", domain_name, "--", "echo", "ssh-ready"]);

match ssh_test {
Ok(output) if output.success() => {
println!("✓ SSH is now available");
return Ok(());
}
Ok(_) => {
// SSH command failed, but that's expected while VM is booting
}
Err(e) => {
println!("SSH test error (expected while booting): {}", e);
}
}

// Check if we've exceeded the timeout
if start_time.elapsed() >= timeout_duration {
return Err(format!("Timeout waiting for SSH after {} seconds", timeout_secs).into());
}

// Wait 5 seconds before next attempt
std::thread::sleep(std::time::Duration::from_secs(5));
}
}

/// Test VM startup and shutdown with libvirt run
fn test_libvirt_run_vm_lifecycle() -> Result<()> {
let bck = get_bck_command()?;
Expand Down Expand Up @@ -664,6 +624,7 @@ fn test_libvirt_run_bind_storage_ro() -> Result<()> {
"--bind-storage-ro",
"--filesystem",
"ext4",
"--ssh-wait",
&test_image,
])
.expect("Failed to run libvirt run with --bind-storage-ro");
Expand Down Expand Up @@ -730,13 +691,10 @@ fn test_libvirt_run_bind_storage_ro() -> Result<()> {
println!("✓ Container storage mount is configured as read-only");
println!("✓ hoststorage tag is present in filesystem configuration");

// Wait for VM to boot and SSH to become available
if let Err(e) = wait_for_ssh_available(&domain_name, 180) {
cleanup_domain(&domain_name);
panic!("Failed to establish SSH connection: {}", e);
}
// SSH is already available due to --ssh-wait flag
println!("✓ SSH is ready (via --ssh-wait)");

// Wait for VM to boot and automatic mount to complete
// Wait for automatic mount to complete
println!("Waiting for VM to boot and automatic mount to complete...");
std::thread::sleep(std::time::Duration::from_secs(10));

Expand Down
72 changes: 71 additions & 1 deletion crates/kit/src/libvirt/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ use crate::libvirt::domain::VirtiofsFilesystem;
use crate::utils::parse_memory_to_mb;
use crate::xml_utils;

/// SSH wait timeout in seconds
const SSH_WAIT_TIMEOUT_SECONDS: u64 = 180;

/// Create a virsh command with optional connection URI
pub(super) fn virsh_command(connect_uri: Option<&str>) -> Result<std::process::Command> {
let mut cmd = std::process::Command::new("virsh");
Expand Down Expand Up @@ -253,6 +256,10 @@ pub struct LibvirtRunOpts {
#[clap(long)]
pub ssh: bool,

/// Wait for SSH to become available and verify connectivity (for testing)
#[clap(long, conflicts_with = "ssh")]
pub ssh_wait: bool,
Comment thread
cgwalters marked this conversation as resolved.

/// Mount host container storage (RO) at /run/host-container-storage
#[clap(long = "bind-storage-ro")]
pub bind_storage_ro: bool,
Expand Down Expand Up @@ -319,6 +326,60 @@ impl LibvirtRunOpts {
}
}

/// Wait for SSH to become available on a libvirt domain
///
/// Polls SSH connectivity by attempting simple commands until successful or timeout.
fn wait_for_ssh_ready(
global_opts: &crate::libvirt::LibvirtOptions,
domain_name: &str,
timeout_secs: u64,
) -> Result<()> {
use std::time::Duration;

debug!(
"Waiting for SSH to become available on domain '{}' (timeout: {}s)",
domain_name, timeout_secs
);

// Create progress bar
let pb = crate::boot_progress::create_boot_progress_bar();
pb.set_message("Waiting for SSH to become available...");

// Clone values for closure
let global_opts_clone = global_opts.clone();
let domain_name_clone = domain_name.to_string();

// Use shared polling function with libvirt-specific test
let (_elapsed, pb) = crate::utils::wait_for_readiness(
pb,
"Waiting for SSH",
|| {
// Create a test SSH connection with short timeout
let ssh_opts = crate::libvirt::ssh::LibvirtSshOpts {
domain_name: domain_name_clone.clone(),
user: "root".to_string(),
command: vec!["true".to_string()], // Simple command to test connectivity
strict_host_keys: false,
timeout: 5, // Short timeout for each attempt
log_level: "ERROR".to_string(),
extra_options: vec![],
suppress_output: true, // Suppress error messages during connectivity testing
};

// Try to connect
match crate::libvirt::ssh::run_ssh_impl(&global_opts_clone, ssh_opts) {
Ok(_) => Ok(true),
Err(_) => Ok(false),
}
},
Duration::from_secs(timeout_secs),
Duration::from_secs(2), // Poll every 2 seconds
)?;

pb.finish_and_clear();
Ok(())
}

/// Execute the libvirt run command
pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtRunOpts) -> Result<()> {
use crate::images;
Expand Down Expand Up @@ -445,12 +506,21 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtRunOpts) -
}
}

if opts.ssh {
if opts.ssh_wait {
// Wait for SSH to be ready and verify connectivity
wait_for_ssh_ready(global_opts, &vm_name, SSH_WAIT_TIMEOUT_SECONDS)?;
println!("Ready; use bcvk libvirt ssh to connect");
Ok(())
} else if opts.ssh {
// Wait for SSH then enter interactive shell
wait_for_ssh_ready(global_opts, &vm_name, SSH_WAIT_TIMEOUT_SECONDS)?;

// Use the libvirt SSH functionality directly
let ssh_opts = crate::libvirt::ssh::LibvirtSshOpts {
domain_name: vm_name,
user: "root".to_string(),
command: vec![],
suppress_output: false,
strict_host_keys: false,
timeout: 30,
log_level: "ERROR".to_string(),
Expand Down
16 changes: 12 additions & 4 deletions crates/kit/src/libvirt/ssh.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ pub struct LibvirtSshOpts {
/// Extra SSH options in key=value format
#[clap(long)]
pub extra_options: Vec<String>,

/// Suppress stdout/stderr output (for connectivity testing)
#[clap(skip)]
pub suppress_output: bool,
}

/// SSH configuration extracted from domain metadata
Expand Down Expand Up @@ -316,13 +320,17 @@ impl LibvirtSshOpts {
.map_err(|e| eyre!("Failed to execute SSH command: {}", e))?;

if !output.stdout.is_empty() {
// Forward stdout to parent process
print!("{}", String::from_utf8_lossy(&output.stdout));
if !self.suppress_output {
// Forward stdout to parent process
print!("{}", String::from_utf8_lossy(&output.stdout));
}
debug!("SSH stdout: {}", String::from_utf8_lossy(&output.stdout));
}
if !output.stderr.is_empty() {
// Forward stderr to parent process
eprint!("{}", String::from_utf8_lossy(&output.stderr));
if !self.suppress_output {
// Forward stderr to parent process
eprint!("{}", String::from_utf8_lossy(&output.stderr));
}
debug!("SSH stderr: {}", String::from_utf8_lossy(&output.stderr));
}

Expand Down
2 changes: 2 additions & 0 deletions crates/kit/src/libvirt/start.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtStartOpts)
timeout: 30,
log_level: "ERROR".to_string(),
extra_options: vec![],
suppress_output: false,
};
return crate::libvirt::ssh::run(global_opts, ssh_opts);
}
Expand Down Expand Up @@ -81,6 +82,7 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, opts: LibvirtStartOpts)
timeout: 30,
log_level: "ERROR".to_string(),
extra_options: vec![],
suppress_output: false,
};
crate::libvirt::ssh::run(global_opts, ssh_opts)
} else {
Expand Down
53 changes: 23 additions & 30 deletions crates/kit/src/run_ephemeral_ssh.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ use color_eyre::Result;
use indicatif::ProgressBar;
use std::os::unix::process::CommandExt;
use std::process::{Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};
use std::time::Duration;
use tracing::debug;

use crate::run_ephemeral::{run_detached, RunEphemeralOpts};
Expand Down Expand Up @@ -116,38 +115,32 @@ pub fn wait_for_ssh_ready(
let timeout = timeout.unwrap_or(SSH_TIMEOUT);
let (_, progress) = wait_for_vm_ssh(container_name, Some(timeout), progress)?;

debug!("Polling SSH connectivity...",);
let start_time = Instant::now();
debug!("Polling SSH connectivity...");

// Use SSH options optimized for connectivity testing
let ssh_options = crate::ssh::SshConnectionOptions::for_connectivity_test();

let mut attempt = 0;
while start_time.elapsed() < timeout {
attempt += 1;
progress.set_message(format!("Polling for SSH readiness (attempt {attempt})"));

// Try to connect via SSH and run a simple command using the centralized SSH function
let status = crate::ssh::connect(
container_name,
vec!["true".to_string()], // Just run 'true' to test connectivity
&ssh_options,
);

if let Ok(exit_status) = status {
if exit_status.success() {
debug!("SSH connection successful, VM is ready");
return Ok((start_time.elapsed(), progress));
let container_name = container_name.to_string();

// Use shared polling function with container-specific test
crate::utils::wait_for_readiness(
progress,
"Waiting for SSH",
|| {
// Try to connect via SSH and run a simple command
let status = crate::ssh::connect(
&container_name,
vec!["true".to_string()], // Just run 'true' to test connectivity
&ssh_options,
);

match status {
Ok(exit_status) if exit_status.success() => Ok(true),
_ => Ok(false),
}
}

thread::sleep(Duration::from_secs(1));
}

Err(color_eyre::eyre::eyre!(
"Timeout waiting for SSH connectivity after {}s",
timeout.as_secs()
))
},
timeout,
Duration::from_secs(1), // Poll every 1 second
)
}

/// Run an ephemeral pod and immediately SSH into it, with lifecycle binding
Expand Down
Loading