Skip to content

Commit 21fa320

Browse files
committed
feat(vmm): add userspace port forwarding for bridge-mode VMs
Introduce dstack-port-forward crate with TCP (splice zero-copy with fallback) and UDP forwarding. Integrate ForwardService into VMM to dynamically manage port forwarding rules when bridge-mode guests report their eth0 IP via the network.eth0 vsock event. Key changes: - New port-forward crate with ForwardService for dynamic rule add/remove - Guest reports eth0 IP in dstack-prepare.sh via dstack-util notify-host - VMM persists guest IP to disk and hot-reloads forwarding rules - On VMM restart, forwarding is restored from persisted guest IPs - guest-agent now includes eth0 interfaces in network info API
1 parent 99b1788 commit 21fa320

13 files changed

Lines changed: 728 additions & 7 deletions

File tree

Cargo.lock

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ members = [
5252
"verifier",
5353
"no_std_check",
5454
"size-parser",
55+
"port-forward",
5556
]
5657
resolver = "2"
5758

@@ -64,6 +65,7 @@ dstack-gateway-rpc = { path = "gateway/rpc" }
6465
dstack-kms-rpc = { path = "kms/rpc" }
6566
dstack-guest-agent-rpc = { path = "guest-agent/rpc" }
6667
dstack-vmm-rpc = { path = "vmm/rpc" }
68+
dstack-port-forward = { path = "port-forward" }
6769
cc-eventlog = { path = "cc-eventlog" }
6870
supervisor = { path = "supervisor" }
6971
supervisor-client = { path = "supervisor/client" }

basefiles/dstack-prepare.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,13 @@ echo "============================"
278278

279279
cd /dstack
280280

281+
# Report eth0 IP address to VMM for bridge-mode port forwarding
282+
ETH0_IP=$(ip -4 -o addr show eth0 2>/dev/null | awk '{print $4}' | cut -d/ -f1 || true)
283+
if [ -n "$ETH0_IP" ]; then
284+
log "Reporting eth0 IP: $ETH0_IP"
285+
dstack-util notify-host -e "network.eth0" -d "$ETH0_IP" || true
286+
fi
287+
281288
if [ $(jq 'has("init_script")' app-compose.json) == true ]; then
282289
log "Running init script"
283290
dstack-util notify-host -e "boot.progress" -d "init-script" || true

guest-agent/src/guest_api_service.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,11 @@ fn get_interfaces() -> Vec<Interface> {
145145
sysinfo::Networks::new_with_refreshed_list()
146146
.into_iter()
147147
.filter_map(|(interface_name, network)| {
148-
if !(interface_name == "dstack-wg0" || interface_name.starts_with("enp")) {
149-
// We only get dstack-wg0 and enp interfaces.
148+
if !(interface_name == "dstack-wg0"
149+
|| interface_name.starts_with("enp")
150+
|| interface_name.starts_with("eth"))
151+
{
152+
// We only get dstack-wg0, enp and eth interfaces.
150153
// Docker bridge is not included due to privacy concerns.
151154
return None;
152155
}

port-forward/Cargo.toml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[package]
2+
name = "dstack-port-forward"
3+
version.workspace = true
4+
edition.workspace = true
5+
6+
[dependencies]
7+
anyhow.workspace = true
8+
tracing.workspace = true
9+
tokio = { workspace = true, features = ["net", "rt", "io-util", "macros", "sync", "time"] }
10+
tokio-util = { version = "0.7", features = ["rt"] }
11+
nix = { workspace = true, features = ["fs", "net", "zerocopy"] }
12+
13+
[dev-dependencies]
14+
tokio = { workspace = true, features = ["full"] }

port-forward/src/lib.rs

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
use std::collections::HashMap;
2+
use std::net::{IpAddr, SocketAddr};
3+
4+
use anyhow::{bail, Result};
5+
use tokio::task::JoinHandle;
6+
use tokio_util::sync::CancellationToken;
7+
8+
mod tcp;
9+
mod udp;
10+
11+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
12+
pub enum Protocol {
13+
Tcp,
14+
Udp,
15+
}
16+
17+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
18+
pub struct ForwardRule {
19+
pub protocol: Protocol,
20+
pub listen_addr: IpAddr,
21+
pub listen_port: u16,
22+
pub target_ip: IpAddr,
23+
pub target_port: u16,
24+
}
25+
26+
impl ForwardRule {
27+
fn listen_sock(&self) -> SocketAddr {
28+
SocketAddr::new(self.listen_addr, self.listen_port)
29+
}
30+
31+
fn target_sock(&self) -> SocketAddr {
32+
SocketAddr::new(self.target_ip, self.target_port)
33+
}
34+
}
35+
36+
struct RunningRule {
37+
cancel: CancellationToken,
38+
task: JoinHandle<()>,
39+
}
40+
41+
/// Manages a dynamic set of port forwarding rules.
42+
///
43+
/// Rules can be added and removed at runtime. Dropping the service cancels all
44+
/// forwarding tasks.
45+
pub struct ForwardService {
46+
cancel: CancellationToken,
47+
rules: HashMap<ForwardRule, RunningRule>,
48+
}
49+
50+
impl ForwardService {
51+
pub fn new() -> Self {
52+
Self {
53+
cancel: CancellationToken::new(),
54+
rules: HashMap::new(),
55+
}
56+
}
57+
58+
/// Add a forwarding rule. Returns error if the rule already exists.
59+
pub fn add_rule(&mut self, rule: ForwardRule) -> Result<()> {
60+
if self.rules.contains_key(&rule) {
61+
bail!("rule already exists: {:?}", rule);
62+
}
63+
64+
let token = self.cancel.child_token();
65+
let listen = rule.listen_sock();
66+
let target = rule.target_sock();
67+
68+
let task = match rule.protocol {
69+
Protocol::Tcp => tokio::spawn(tcp::run_tcp_forwarder(listen, target, token.clone())),
70+
Protocol::Udp => tokio::spawn(udp::run_udp_forwarder(listen, target, token.clone())),
71+
};
72+
73+
tracing::info!("added forwarding rule: {listen} -> {target} ({:?})", rule.protocol);
74+
self.rules.insert(rule, RunningRule { cancel: token, task });
75+
Ok(())
76+
}
77+
78+
/// Remove a forwarding rule and stop its task.
79+
pub async fn remove_rule(&mut self, rule: &ForwardRule) -> Result<()> {
80+
match self.rules.remove(rule) {
81+
Some(running) => {
82+
running.cancel.cancel();
83+
let _ = running.task.await;
84+
tracing::info!(
85+
"removed forwarding rule: {} -> {} ({:?})",
86+
rule.listen_sock(),
87+
rule.target_sock(),
88+
rule.protocol,
89+
);
90+
Ok(())
91+
}
92+
None => bail!("rule not found: {:?}", rule),
93+
}
94+
}
95+
96+
/// Number of active rules.
97+
pub fn len(&self) -> usize {
98+
self.rules.len()
99+
}
100+
101+
pub fn is_empty(&self) -> bool {
102+
self.rules.is_empty()
103+
}
104+
105+
/// Gracefully stop all forwarding and wait for tasks to finish.
106+
pub async fn shutdown(mut self) {
107+
self.cancel.cancel();
108+
for (_, running) in std::mem::take(&mut self.rules) {
109+
let _ = running.task.await;
110+
}
111+
}
112+
}
113+
114+
impl Default for ForwardService {
115+
fn default() -> Self {
116+
Self::new()
117+
}
118+
}
119+
120+
impl Drop for ForwardService {
121+
fn drop(&mut self) {
122+
self.cancel.cancel();
123+
}
124+
}

0 commit comments

Comments
 (0)