diff --git a/docs/usage.md b/docs/usage.md index 07c29f8..78bb428 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -85,18 +85,52 @@ The `virtio-net` option adds a network interface to a virtual machine. #### Arguments +Arguments to create a virtio-net device that has offloading enabled by default and will send a VFKIT magic +value after establishing the connection: - `unixSocketPath`: Path to a UNIX socket to attach to the guest network interface. - `mac`: MAC address of a virtual machine. +Arguments to create a virtio-net device with a unix datagram or unix stream socket backend: +- `type`: Unix socket type: + - `unixgram`: unix datagram socket-based backend such as gvproxy or vmnet-helper. + - `unixstream`: unix stream socket-based userspace network proxy such as passt or socket_vmnet. +- `path`: Unix socket path. Mutually exclusive with the `fd=` option. +- `fd`: Unix socket file descriptor. Mutually exclusive with the `path=` option. +- `mac`: MAC address of a virtual machine. +- `offloading`: (Optional) Whether or not to enable network offloading between the guest and host. +Default value is `off`. +- `vfkitMagic`: (Optional) Whether to send the vfkit magic value after establishing a network connection. +Default value is `off`. Only supported with the `unixgram` type. + +> [!NOTE] +> The `unixSocketPath`, `type={unixgram, unixstream}` arguments are mutually exclusive and cannot be used together. + #### Example -This adds a virtio-net device to a virtual machine and redirects all guest network traffic to the corresponding -socket at `/Users/user/vm-network.sock` with a MAC address of `ff:ff:ff:ff:ff:ff`: +If you want to use a tool like vmnet-helper as your backend, you'll be interested in using the `type=unixgram` +argument. This will create a virtio-net device and redirect all guest network traffic to the corresponding socket. +Here, we're not going to use any of the optional arguments. Therefore, offloading will be disabled by default +and the vfkit magic value won't be sent when the connection is established: + +``` +--device virtio-net,type=unixgram,path=/Users/user/vm-network.sock,mac=ff:ff:ff:ff:ff:ff +``` + +If you want to use gvproxy instead, you're going to want to use some of the optional arguments krunkit provides. +We're going to want to enable offloading, and if gvproxy is running in vfkit mode, we'll also want to send the +vfkit magic value when the connection establishes: ``` ---device virtio-net,unixSocketPath=/Users/user/vm-network.sock,mac=ff:ff:ff:ff:ff:ff +--device virtio-net,type=unixgram,path=/Users/user/vm-network.sock,mac=ff:ff:ff:ff:ff:ff,offloading=on,vfkitMagic=on ``` +You can also use a network proxy such as passt by using the `type=unixstream` argument: +``` +--device virtio-net,type=unixstream,fd=,mac=ff:ff:ff:ff:ff:ff,offloading=true +``` + +To see performance implications of choosing offloading vs. not offloading, see [this table](#offloading-performance-implications) + ### Serial Port The `virtio-serial` option adds a serial device to a virtual machine. This allows for redirection of virtual @@ -174,3 +208,25 @@ Response: `VirtualMachineState{Running, Stopped}` `POST /vm/state` `{ "state": "Stop" }` Response: `VirtualMachineStateStopped` + +## Offloading Performance Implications + +The table below provides some data on how offloading effects the gvproxy and vmnet-helper backends: + +#### vment-helper offloading + +| network | vm | offloading | iper3 | iperf3 -R | +|---------------|--------- |------------|---------------|---------------| +| vmnet-helper | krunkit | true | 1.38 Gbits/s | 46.20 Gbits/s | +| vmnet-helper | krunkit | false | 10.10 Gbits/s | 8.38 Gbits/s | +| vmnet-helper | vfkit | true | 4.27 Gbits/s | 8.09 Gbits/s | +| vmnet-helper | vfkit | false | 10.70 Gbits/s | 8.41 Gbits/s | + +#### gvproxy offloading + +| network | vm | offloading | iper3 | iperf3 -R | +|---------------|--------- |------------|---------------|---------------| +| gvproxy | krunkit | true | 1.40 Gbits/s | 20.00 Gbits/s | +| gvproxy | krunkit | false | 1.47 Gbits/s | 2.58 Gbits/s | +| gvproxy | vfkit | false | 1.43 Gbits/s | 2.84 Gbits/s | + diff --git a/src/cmdline.rs b/src/cmdline.rs index d861c3f..056315e 100644 --- a/src/cmdline.rs +++ b/src/cmdline.rs @@ -2,7 +2,12 @@ use crate::{status::RestfulUri, virtio::VirtioDeviceConfig}; -use std::{collections::HashMap, path::PathBuf, str::FromStr}; +use std::{ + collections::HashMap, + ffi::{c_char, CString}, + path::PathBuf, + str::FromStr, +}; use anyhow::{anyhow, Context, Result}; use clap::Parser; @@ -102,6 +107,26 @@ pub fn check_unknown_args(args: HashMap, label: &str) -> Result< Ok(()) } +/// Parse a string slice and convert it to a boolean if possible. In addition to "true" and "false" +/// being valid strings, "on" and "off" are also valid. +pub fn parse_boolean(value: &str) -> Result { + match value { + "true" | "on" => Ok(true), + "false" | "off" => Ok(false), + _ => Err(anyhow!("invalid boolean value {value}")), + } +} + +/// Convert a CString value to a pointer. If the string is empty, the function will return a NULL +/// ptr. +pub fn cstring_to_ptr(value: &CString) -> *const c_char { + if value.is_empty() { + std::ptr::null() + } else { + value.as_ptr() + } +} + /// A wrapper of all data associated with the bootloader argument. mod bootloader { use super::*; @@ -380,6 +405,16 @@ mod tests { "virtio-gpu,width=800,height=600", "--device", "virtio-input,keyboard", + "--device", + "virtio-net,type=unixgram,path=/Users/user/net.sock,mac=00:00:00:00:00:00,offloading=true,vfkitMagic=off", + "--device", + "virtio-net,type=unixgram,fd=4,mac=00:00:00:00:00:00", + "--device", + "virtio-net,type=unixstream,path=/Users/user/net.sock,mac=00:00:00:00:00:00,offloading=on", + "--device", + "virtio-net,type=unixstream,fd=4,mac=00:00:00:00:00:00,offloading=off", + "--device", + "virtio-net,type=unixstream,fd=4,mac=00:00:00:00:00:00", "--restful-uri", "tcp://localhost:49573", "--gui", @@ -389,6 +424,102 @@ mod tests { let mut args = Args::try_parse_from(cmdline).unwrap(); + let net = args + .devices + .pop() + .expect("expected 15th virtio device config"); + if let VirtioDeviceConfig::Net(net) = net { + if let SocketType::UnixStream = net.socket_type { + assert_eq!(net.socket_config.path, None,); + assert_eq!(net.socket_config.fd, Some(4)); + assert_eq!(net.socket_config.offloading, false); + assert_eq!(net.socket_config.send_vfkit_magic, false); + } else { + panic!("expected virtio-net device to use the unixstream argument"); + } + assert_eq!(net.mac_address, MacAddress::new([0, 0, 0, 0, 0, 0])); + } else { + panic!("expected virtio-net device as 15th device config argument"); + } + + let net = args + .devices + .pop() + .expect("expected 14th virtio device config"); + if let VirtioDeviceConfig::Net(net) = net { + if let SocketType::UnixStream = net.socket_type { + assert_eq!(net.socket_config.path, None,); + assert_eq!(net.socket_config.fd, Some(4)); + assert_eq!(net.socket_config.offloading, false); + assert_eq!(net.socket_config.send_vfkit_magic, false); + } else { + panic!("expected virtio-net device to use the unixstream argument"); + } + assert_eq!(net.mac_address, MacAddress::new([0, 0, 0, 0, 0, 0])); + } else { + panic!("expected virtio-net device as 14th device config argument"); + } + + let net = args + .devices + .pop() + .expect("expected 13th virtio device config"); + if let VirtioDeviceConfig::Net(net) = net { + if let SocketType::UnixStream = net.socket_type { + assert_eq!( + net.socket_config.path, + Some(PathBuf::from_str("/Users/user/net.sock").unwrap()) + ); + assert_eq!(net.socket_config.fd, None); + assert_eq!(net.socket_config.offloading, true); + assert_eq!(net.socket_config.send_vfkit_magic, false); + } else { + panic!("expected virtio-net device to use the unixstream argument"); + } + assert_eq!(net.mac_address, MacAddress::new([0, 0, 0, 0, 0, 0])); + } else { + panic!("expected virtio-net device as 13th device config argument"); + } + + let net = args + .devices + .pop() + .expect("expected 12th virtio device config"); + if let VirtioDeviceConfig::Net(net) = net { + if let SocketType::UnixGram = net.socket_type { + assert_eq!(net.socket_config.path, None); + assert_eq!(net.socket_config.fd, Some(4)); + assert_eq!(net.socket_config.offloading, false); + assert_eq!(net.socket_config.send_vfkit_magic, false); + } else { + panic!("expected virtio-net device to use the unixgram argument"); + } + assert_eq!(net.mac_address, MacAddress::new([0, 0, 0, 0, 0, 0])); + } else { + panic!("expected virtio-net device as 12th device config argument"); + } + + let net = args + .devices + .pop() + .expect("expected 11th virtio device config"); + if let VirtioDeviceConfig::Net(net) = net { + if let SocketType::UnixGram = net.socket_type { + assert_eq!( + net.socket_config.path, + Some(PathBuf::from_str("/Users/user/net.sock").unwrap()) + ); + assert_eq!(net.socket_config.fd, None); + assert_eq!(net.socket_config.offloading, true); + assert_eq!(net.socket_config.send_vfkit_magic, false); + } else { + panic!("expected virtio-net device to use the unixgram argument"); + } + assert_eq!(net.mac_address, MacAddress::new([0, 0, 0, 0, 0, 0])); + } else { + panic!("expected virtio-net device as 11th device config argument"); + } + let input = args .devices .pop() @@ -441,10 +572,17 @@ mod tests { .pop() .expect("expected 6th virtio device config"); if let VirtioDeviceConfig::Net(net) = net { - assert_eq!( - net.unix_socket_path, - PathBuf::from_str("/Users/user/net.sock").unwrap() - ); + if let SocketType::UnixGram = net.socket_type { + assert_eq!( + net.socket_config.path, + Some(PathBuf::from_str("/Users/user/net.sock").unwrap()) + ); + assert_eq!(net.socket_config.fd, None); + assert_eq!(net.socket_config.offloading, true); + assert_eq!(net.socket_config.send_vfkit_magic, true); + } else { + panic!("expected virtio-net device to use the unixSocketPath argument"); + } assert_eq!(net.mac_address, MacAddress::new([0, 0, 0, 0, 0, 0])); } else { panic!("expected virtio-net device as 6th device config argument"); diff --git a/src/virtio.rs b/src/virtio.rs index 632194f..2a9436e 100644 --- a/src/virtio.rs +++ b/src/virtio.rs @@ -1,9 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 -use crate::cmdline::{check_required_args, check_unknown_args, parse_args}; +use crate::cmdline::{ + check_required_args, check_unknown_args, cstring_to_ptr, parse_args, parse_boolean, +}; use std::{ - ffi::{c_char, CString}, + ffi::{c_char, c_int, CString}, + os::fd::RawFd, os::unix::ffi::OsStrExt, path::{Path, PathBuf}, str::FromStr, @@ -12,6 +15,32 @@ use std::{ use anyhow::{anyhow, Context, Result}; use mac_address::MacAddress; +/// Taken from https://github.com/containers/libkrun/blob/7116644749c7b1028a970c9e8bd2d0163745a225/include/libkrun.h#L269 +const NET_FEATURE_CSUM: u32 = 1 << 0; +const NET_FEATURE_GUEST_CSUM: u32 = 1 << 1; +const NET_FEATURE_GUEST_TSO4: u32 = 1 << 7; +const NET_FEATURE_GUEST_TSO6: u32 = 1 << 8; +const NET_FEATURE_GUEST_UFO: u32 = 1 << 10; +const NET_FEATURE_HOST_TSO4: u32 = 1 << 11; +const NET_FEATURE_HOST_TSO6: u32 = 1 << 12; +const NET_FEATURE_HOST_UFO: u32 = 1 << 14; + +/// These are the features enabled by krun_set_passt_fd and krun_set_gvproxy_path. +const COMPAT_NET_FEATURES: u32 = NET_FEATURE_CSUM + | NET_FEATURE_GUEST_CSUM + | NET_FEATURE_GUEST_TSO4 + | NET_FEATURE_GUEST_UFO + | NET_FEATURE_HOST_TSO4 + | NET_FEATURE_HOST_UFO; + +/// Send the VFKIT magic after establishing the connection, +/// as required by gvproxy in vfkit mode. +const NET_FLAG_VFKIT: u32 = 1 << 0; + +const SOCK_TYPE_UNIX_SOCKET_PATH: &str = "unixSocketPath"; +const SOCK_TYPE_UNIXGRAM: &str = "unixgram"; +const SOCK_TYPE_UNIXSTREAM: &str = "unixstream"; + #[link(name = "krun-efi")] extern "C" { fn krun_add_disk2( @@ -23,9 +52,23 @@ extern "C" { ) -> i32; fn krun_add_vsock_port(ctx_id: u32, port: u32, c_filepath: *const c_char) -> i32; fn krun_add_virtiofs(ctx_id: u32, c_tag: *const c_char, c_path: *const c_char) -> i32; - fn krun_set_gvproxy_path(ctx_id: u32, c_path: *const c_char) -> i32; - fn krun_set_net_mac(ctx_id: u32, c_mac: *const u8) -> i32; fn krun_set_console_output(ctx_id: u32, c_filepath: *const c_char) -> i32; + fn krun_add_net_unixgram( + ctx_id: u32, + c_path: *const c_char, + fd: c_int, + c_mac: *const u8, + features: u32, + flags: u32, + ) -> i32; + fn krun_add_net_unixstream( + ctx_id: u32, + c_path: *const c_char, + fd: c_int, + c_mac: *const u8, + features: u32, + flags: u32, + ) -> i32; } #[repr(u32)] @@ -296,11 +339,32 @@ impl FromStr for VsockAction { } } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SocketConfig { + pub path: Option, + pub fd: Option, + pub offloading: bool, + pub send_vfkit_magic: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SocketType { + /// Socket type for creating an independent virtio-net device with a + /// unixgram-based backend. + UnixGram, + /// Socket type for creating an independent virtio-net device with a + /// unixstream-based backend. + UnixStream, +} + /// Configuration of a virtio-net device. -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct NetConfig { - /// Path to underlying gvproxy socket. - pub unix_socket_path: PathBuf, + /// Socket type. + pub socket_type: SocketType, + + /// Socket config information. + pub socket_config: SocketConfig, /// Network MAC address. pub mac_address: MacAddress, @@ -310,17 +374,51 @@ impl FromStr for NetConfig { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let mut net_config = Self::default(); let mut args = parse_args(s.to_string())?; - check_required_args(&args, "virtio-net", &["unixSocketPath", "mac"])?; + check_required_args(&args, "virtio-net", &["mac"])?; + + let mut socket_type: Option = None; + let mut socket_config = SocketConfig::default(); + + if let Some(path) = args.remove(SOCK_TYPE_UNIX_SOCKET_PATH) { + let path = PathBuf::from_str(path.as_str()) + .context("virtio-net \"unixSocketPath\" is not a valid path")?; + socket_config = SocketConfig { + path: Some(path), + fd: None, + offloading: true, + send_vfkit_magic: true, + }; + + socket_type = Some(SocketType::UnixGram); + } + + if let Some(t) = args.remove("type") { + if socket_type.is_some() { + return Err(anyhow!( + "virtio-net \"type\" and \"unixSocketPath\" are mutually exclusive and cannot be used together." + )); + } - let unix_socket_path = args.remove("unixSocketPath").unwrap(); - net_config.unix_socket_path = PathBuf::from_str(unix_socket_path.as_str()) - .context("unixSocketPath argument not a valid path")?; + let (_type, _conf) = parse_socket_config(&t, &mut args)?; + socket_config = _conf; + socket_type = Some(_type); + } + + if socket_type.is_none() { + return Err(anyhow!( + "virtio-net device is missing \"type\" or \"unixSocketPath\"" + )); + }; let mac = args.remove("mac").unwrap(); - net_config.mac_address = MacAddress::from_str(mac.as_str()) - .context("unable to parse mac address from argument")?; + + let net_config = NetConfig { + socket_type: socket_type.unwrap(), + socket_config, + mac_address: MacAddress::from_str(mac.as_str()) + .context("virtio-net unable to parse address from \"mac\" argument")?, + }; check_unknown_args(args, "virtio-net")?; @@ -328,24 +426,125 @@ impl FromStr for NetConfig { } } -/// Set the gvproxy's path and network MAC address. -impl KrunContextSet for NetConfig { - unsafe fn krun_ctx_set(&self, id: u32) -> Result<(), anyhow::Error> { - let path_cstr = path_to_cstring(&self.unix_socket_path)?; - let mac = self.mac_address.bytes(); +fn parse_socket_config( + socket_type: &str, + args: &mut std::collections::HashMap, +) -> Result<(SocketType, SocketConfig), anyhow::Error> { + let mut socket_config = SocketConfig::default(); - if krun_set_gvproxy_path(id, path_cstr.as_ptr()) < 0 { - return Err(anyhow!(format!( - "unable to set gvproxy path {}", - &self.unix_socket_path.display() - ))); + let socket_type = match socket_type { + SOCK_TYPE_UNIXGRAM => SocketType::UnixGram, + SOCK_TYPE_UNIXSTREAM => SocketType::UnixStream, + _ => { + return Err(anyhow!( + "virtio-net unsupported \"type\" input {socket_type}" + )) } + }; - if krun_set_net_mac(id, mac.as_ptr()) < 0 { - return Err(anyhow!(format!( - "unable to set net MAC address {}", - self.mac_address - ))); + if let Some(path) = args.remove("path") { + socket_config.path = Some( + PathBuf::from_str(path.as_ref()).context("virtio-net \"path\" is not a valid path")?, + ); + } + + if let Some(fd) = args.remove("fd") { + socket_config.fd = Some( + fd.parse::() + .context("virtio-net unable to convert \"fd\" value {fd} to a file descriptor")?, + ); + } + + if socket_config.fd.is_some() && socket_config.path.is_some() { + return Err(anyhow!( + "virtio-net device arguments \"path\" and \"fd\" are mutually exclusive and cannot be used together" + )); + } + + if let Some(offloading) = args.remove("offloading") { + socket_config.offloading = parse_boolean(&offloading)?; + } + + if let Some(send_vfkit_magic) = args.remove("vfkitMagic") { + if socket_type != SocketType::UnixGram { + return Err(anyhow!( + "virtio-net only \"type=unixgram\" supports the vfkitMagic argument" + )); + } + socket_config.send_vfkit_magic = parse_boolean(&send_vfkit_magic)?; + } + + Ok((socket_type, socket_config)) +} + +/// Set the gvproxy's path and network MAC address. +impl KrunContextSet for NetConfig { + unsafe fn krun_ctx_set(&self, id: u32) -> Result<(), anyhow::Error> { + match &self.socket_type { + SocketType::UnixGram => { + let features = if self.socket_config.offloading { + COMPAT_NET_FEATURES + } else { + 0 + }; + + let path = match &self.socket_config.path { + Some(path) => path_to_cstring(path)?, + None => path_to_cstring(&PathBuf::new())?, + }; + + let flags = if self.socket_config.send_vfkit_magic { + NET_FLAG_VFKIT + } else { + 0 + }; + + if krun_add_net_unixgram( + id, + cstring_to_ptr(&path), + self.socket_config.fd.unwrap_or(-1), + self.mac_address.bytes().as_ptr(), + features, + flags, + ) < 0 + { + // TODO(jakecorrenti): if this fails, we should display all of the values the + // user provided to the virtio-net cmdline + return Err(anyhow!(format!( + "virtio-net unable to add device with unix datagram backend {:#?}", + self.socket_config + ))); + } + } + SocketType::UnixStream => { + let features = if self.socket_config.offloading { + COMPAT_NET_FEATURES + } else { + 0 + }; + + let path = match &self.socket_config.path { + Some(path) => path_to_cstring(path)?, + None => path_to_cstring(&PathBuf::new())?, + }; + + if krun_add_net_unixstream( + id, + cstring_to_ptr(&path), + self.socket_config.fd.unwrap_or(-1), + self.mac_address.bytes().as_ptr(), + features, + 0, + ) < 0 + { + // TODO(jakecorrenti): if this fails, we should display all of the values the + // user provided to the virtio-net cmdline + return Err(anyhow!(format!( + "virtio-net unable to add device with unix stream backend {:#?}", + self.socket_config + ))); + } + } } Ok(())