docs(vmm): update bridge networking guide with DHCP notification and firewall rules

kvinwang · kvinwang · commit 054e088f2957 · 2026-02-02T07:47:34.000Z
Add dhcp-notify.sh script integration, nftables firewall rules for
standalone bridges, and setup-bridge.sh improvements.
diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md
@@ -29,7 +29,7 @@ Only the mode is per-VM; the bridge interface name always comes from the global
 
 ## Host setup
 
-### Option A: Using libvirt default network (recommended)
+### Option A: Using libvirt default network
 
 libvirt's default network provides a bridge (`virbr0`) with DHCP (dnsmasq) and NAT out of the box.
 
@@ -88,20 +88,64 @@ sudo sysctl -p /etc/sysctl.d/99-dstack-bridge.conf
 
 ```bash
 sudo apt install -y dnsmasq
+```
+
+Install the DHCP notification script (notifies VMM when a VM gets an IP so port forwarding can be established):
+
+```bash
+sudo cp scripts/dhcp-notify.sh /usr/local/bin/dhcp-notify.sh
+sudo chmod +x /usr/local/bin/dhcp-notify.sh
+```
+
+Create dnsmasq config:
 
+```ini
 # /etc/dnsmasq.d/dstack-br0.conf
 interface=dstack-br0
 bind-interfaces
 dhcp-range=10.0.100.10,10.0.100.254,255.255.255.0,12h
 dhcp-option=option:router,10.0.100.1
 dhcp-option=option:dns-server,8.8.8.8,1.1.1.1
+dhcp-script=/usr/local/bin/dhcp-notify.sh
 ```
 
+The `dhcp-script` option tells dnsmasq to call the notification script on every lease event. The script sends the MAC and IP to VMM's `ReportDhcpLease` RPC, which triggers automatic port forwarding for the VM.
+
 ```bash
 sudo systemctl restart dnsmasq
 ```
 
-**4. Update vmm.toml:**
+**4. Firewall rules (nftables):**
+
+When the host firewall has a restrictive INPUT policy (e.g. `drop`), the bridge's DHCP and DNS traffic will be silently blocked. libvirt handles this automatically for virbr0, but a standalone bridge needs explicit rules.
+
+```bash
+BRIDGE=dstack-br0
+SUBNET=10.0.100.0/24
+
+# Allow DHCP and DNS from VMs (INPUT/OUTPUT)
+sudo nft add rule ip filter INPUT iifname "$BRIDGE" udp dport 67 counter accept
+sudo nft add rule ip filter INPUT iifname "$BRIDGE" udp dport 53 counter accept
+sudo nft add rule ip filter INPUT iifname "$BRIDGE" tcp dport 53 counter accept
+sudo nft add rule ip filter OUTPUT oifname "$BRIDGE" udp dport 68 counter accept
+sudo nft add rule ip filter OUTPUT oifname "$BRIDGE" udp dport 53 counter accept
+
+# Allow forwarding for VM traffic
+sudo nft add rule ip filter FORWARD ip saddr "$SUBNET" iifname "$BRIDGE" counter accept
+sudo nft add rule ip filter FORWARD ip daddr "$SUBNET" oifname "$BRIDGE" ct state related,established counter accept
+sudo nft add rule ip filter FORWARD iifname "$BRIDGE" oifname "$BRIDGE" counter accept
+
+# NAT masquerade for outbound traffic
+sudo nft add rule ip nat POSTROUTING ip saddr "$SUBNET" ip daddr 224.0.0.0/24 counter return
+sudo nft add rule ip nat POSTROUTING ip saddr "$SUBNET" ip daddr 255.255.255.255 counter return
+sudo nft add rule ip nat POSTROUTING ip saddr "$SUBNET" ip daddr != "$SUBNET" counter masquerade
+```
+
+If the host uses libvirt, nftables rules may be in custom chains (`LIBVIRT_INP`, `LIBVIRT_FWO`, etc.) instead of the default `INPUT`/`FORWARD` chains. Adjust the chain names accordingly.
+
+To make these rules persistent across reboots, save them with `nft list ruleset > /etc/nftables.conf` or add them to a systemd service.
+
+**5. Update vmm.toml:**
 
 ```toml
 [cvm.networking]
@@ -127,10 +171,25 @@ sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper
 
 - VMM passes `-netdev bridge,id=net0,br=<bridge>` to QEMU
 - QEMU's bridge helper (setuid) creates a TAP device and attaches it to the bridge
-- Guest MAC address is derived from SHA256 of the VM ID (stable across restarts for DHCP IP consistency)
+- Guest MAC address is derived from SHA256 of the VM ID, with an optional configurable prefix (stable across restarts for DHCP IP consistency)
+- The host DHCP server (dnsmasq) assigns an IP and calls `dhcp-notify.sh`, which notifies VMM via the `ReportDhcpLease` RPC
+- VMM matches the MAC address to identify the VM and establishes port forwarding rules
 - When QEMU exits, the TAP device is automatically destroyed
 - VMM does not need root or `CAP_NET_ADMIN`
 
+### MAC address prefix
+
+You can configure a fixed MAC address prefix (0–3 bytes) in vmm.toml:
+
+```toml
+[cvm.networking]
+mode = "bridge"
+bridge = "dstack-br0"
+mac_prefix = "52:54:00"
+```
+
+The remaining bytes are derived from the VM ID hash. The prefix applies to all networking modes, not just bridge. The locally-administered bit is always set on the first byte.
+
 ## Operational notes
 
 ### Do not restart the bridge while VMs are running
@@ -139,9 +198,10 @@ sudo chmod u+s /usr/lib/qemu/qemu-bridge-helper
 
 ### Firewall considerations
 
-- libvirt injects nftables rules for NAT masquerade and forwarding automatically
-- If using a manual bridge, ensure your firewall allows forwarding for the bridge subnet and has masquerade rules for outbound NAT
+- libvirt automatically injects nftables rules for INPUT (DHCP/DNS), FORWARD, and NAT masquerade into its own chains (`LIBVIRT_INP`, `LIBVIRT_FWO`, `LIBVIRT_FWI`, `LIBVIRT_PRT`)
+- A standalone bridge requires **all** of these rules to be added manually (see Option B step 4 above). The most common failure mode is a restrictive INPUT policy silently dropping DHCP requests from VMs — if VMs on a custom bridge don't get an IP, check `sudo nft list chain ip filter INPUT` first
 - Docker's nftables chains (`DOCKER-FORWARD`) run before libvirt's but do not block virbr0 traffic
+- Use `setup-bridge.sh check --bridge <name>` to diagnose missing rules
 
 ### Mixing networking modes
 
diff --git a/scripts/dhcp-notify.sh b/scripts/dhcp-notify.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network>
+# SPDX-License-Identifier: Apache-2.0
+#
+# DHCP lease notification script for dnsmasq.
+#
+# Called by dnsmasq via --dhcp-script on lease events.
+# Notifies dstack-vmm of the guest's MAC and IP so that port
+# forwarding can be established automatically.
+#
+# Arguments (set by dnsmasq):
+#   $1  action   add | del | old
+#   $2  mac      MAC address of the guest NIC
+#   $3  ip       IPv4 address assigned by DHCP
+#   $4  hostname (optional)
+#
+# Configuration:
+#   VMM_URL  Base URL of dstack-vmm (default: http://127.0.0.1:9080)
+
+ACTION="$1"
+MAC="$2"
+IP="$3"
+
+VMM_URL="${VMM_URL:-http://127.0.0.1:9080}"
+
+logger -t dhcp-notify "action=$ACTION mac=$MAC ip=$IP"
+
+case "$ACTION" in
+    add|old)
+        curl -s -X POST "${VMM_URL}/prpc/ReportDhcpLease" \
+            -H 'Content-Type: application/json' \
+            -d "{\"mac\":\"$MAC\",\"ip\":\"$IP\"}" \
+            || logger -t dhcp-notify "failed to notify VMM"
+        ;;
+    del)
+        # Could clear forwarding on lease expiry; not implemented yet.
+        ;;
+esac
diff --git a/scripts/setup-bridge.sh b/scripts/setup-bridge.sh
@@ -192,6 +192,47 @@ check_dhcp() {
     check_info "Run: $(basename "$0") setup --mode standalone --bridge $BRIDGE"
 }
 
+check_dhcp_notify() {
+    echo
+    bold "DHCP lease notification"
+
+    local provider
+    provider=$(detect_bridge_provider)
+
+    if [[ "$provider" == libvirt:* ]]; then
+        check_warn "libvirt DHCP does not support dhcp-script callback"
+        check_info "port forwarding requires manual PRPC call or alternative notification"
+        return
+    fi
+
+    # Check dnsmasq config for dhcp-script
+    local conf_files=(/etc/dnsmasq.d/*"$BRIDGE"* /etc/dnsmasq.d/*.conf)
+    local found_script=""
+    for f in "${conf_files[@]}"; do
+        [[ -f "$f" ]] || continue
+        local script_path
+        script_path=$(grep -oP '^dhcp-script=\K.*' "$f" 2>/dev/null || true)
+        if [[ -n "$script_path" ]]; then
+            found_script="$script_path"
+            break
+        fi
+    done
+
+    if [[ -z "$found_script" ]]; then
+        check_warn "no dhcp-script configured in dnsmasq"
+        check_info "port forwarding will not be set up automatically"
+        check_info "add 'dhcp-script=/usr/local/bin/dhcp-notify.sh' to dnsmasq config"
+        return
+    fi
+
+    if [[ -x "$found_script" ]]; then
+        check_pass "dhcp-script configured: $found_script"
+    else
+        check_fail "dhcp-script $found_script is not executable or missing"
+        check_info "Fix: sudo chmod +x $found_script"
+    fi
+}
+
 check_ip_forward() {
     echo
     bold "IP forwarding"
@@ -254,6 +295,45 @@ print(iface.network)
     check_info "For standalone: systemd-networkd IPMasquerade=both or manual nftables rules."
 }
 
+check_dhcp_firewall() {
+    echo
+    bold "DHCP/DNS firewall rules"
+
+    local nft_rules=""
+    if command -v nft &>/dev/null; then
+        nft_rules=$(sudo nft list ruleset 2>/dev/null || true)
+    fi
+
+    if [[ -z "$nft_rules" ]]; then
+        check_warn "nft not available, cannot check firewall rules"
+        return
+    fi
+
+    # Check INPUT policy
+    local input_policy
+    input_policy=$(echo "$nft_rules" | grep -A1 'chain INPUT' | grep -oP 'policy \K\w+' | head -1 || echo "")
+
+    if [[ "$input_policy" == "accept" ]]; then
+        check_pass "INPUT policy is accept (DHCP/DNS allowed by default)"
+        return
+    fi
+
+    # Restrictive INPUT policy — need explicit rules
+    if echo "$nft_rules" | grep -q "iifname \"$BRIDGE\".*udp dport 67.*accept"; then
+        check_pass "DHCP input rule for $BRIDGE"
+    else
+        check_fail "no DHCP input rule for $BRIDGE (INPUT policy: ${input_policy:-unknown})"
+        check_info "VMs will not get DHCP leases without this rule"
+        check_info "Fix: sudo nft add rule ip filter INPUT iifname \"$BRIDGE\" udp dport 67 counter accept"
+    fi
+
+    if echo "$nft_rules" | grep -q "iifname \"$BRIDGE\".*udp dport 53.*accept"; then
+        check_pass "DNS input rule for $BRIDGE"
+    else
+        check_warn "no DNS input rule for $BRIDGE"
+    fi
+}
+
 check_forward_rules() {
     echo
     bold "forwarding rules"
@@ -462,21 +542,126 @@ sudo mv /tmp/.dstack-br-network $network"
             dhcp_end="${prefix}.254"
         fi
 
+        # Install dhcp-notify.sh if present
+        local notify_script="/usr/local/bin/dhcp-notify.sh"
+        local dhcp_script_line=""
+        local src_notify
+        src_notify="$(cd "$(dirname "$0")" && pwd)/dhcp-notify.sh"
+        if [[ -f "$src_notify" ]]; then
+            run_cmd sudo cp "$src_notify" "$notify_script"
+            run_cmd sudo chmod +x "$notify_script"
+            dhcp_script_line="dhcp-script=${notify_script}"
+            echo "  installed $notify_script"
+        else
+            echo "  $(yellow '[WARN]') dhcp-notify.sh not found at $src_notify"
+            echo "  VM port forwarding will not be set up automatically"
+        fi
+
         run_cmd bash -c "cat > /tmp/.dstack-dnsmasq <<HEREDOC
 interface=$BRIDGE
 bind-interfaces
 dhcp-range=${dhcp_start},${dhcp_end},255.255.255.0,12h
 dhcp-option=option:router,${bridge_ip}
 dhcp-option=option:dns-server,8.8.8.8,1.1.1.1
+${dhcp_script_line}
 HEREDOC
 sudo mv /tmp/.dstack-dnsmasq $conf"
         echo "  created $conf"
         run_cmd sudo systemctl restart dnsmasq
         echo "  restarted dnsmasq"
     fi
 
+    # 3. Firewall rules for standalone bridge
+    setup_standalone_firewall
+
+    echo
+    echo "  standalone provides: bridge, DHCP (dnsmasq), NAT, firewall rules"
+}
+
+# --- Setup: standalone firewall ---
+
+setup_standalone_firewall() {
     echo
-    echo "  standalone provides: bridge, DHCP (dnsmasq), NAT (systemd-networkd IPMasquerade)"
+    bold "Setting up firewall rules for $BRIDGE"
+
+    local subnet
+    subnet=$(ip -4 -o addr show "$BRIDGE" 2>/dev/null | awk '{print $4}' | head -1)
+    if [[ -z "$subnet" ]]; then
+        echo "  $(red 'ERROR'): bridge $BRIDGE has no IP, cannot configure firewall"
+        return 1
+    fi
+
+    local net_cidr
+    net_cidr=$(python3 -c "
+import ipaddress
+iface = ipaddress.ip_interface('$subnet')
+print(iface.network)
+" 2>/dev/null || echo "")
+
+    if [[ -z "$net_cidr" ]]; then
+        echo "  $(red 'ERROR'): cannot parse subnet $subnet"
+        return 1
+    fi
+
+    if ! command -v nft &>/dev/null; then
+        echo "  $(yellow '[WARN]') nft not found, skipping firewall setup"
+        echo "  you may need to configure iptables manually"
+        return 0
+    fi
+
+    # Detect whether to use libvirt chains or default chains
+    local inp_chain="INPUT"
+    local out_chain="OUTPUT"
+    local fwd_chain="FORWARD"
+    local nat_chain="POSTROUTING"
+
+    if sudo nft list chain ip filter LIBVIRT_INP &>/dev/null 2>&1; then
+        inp_chain="LIBVIRT_INP"
+        out_chain="LIBVIRT_OUT"
+        echo "  using libvirt filter chains (LIBVIRT_INP/LIBVIRT_OUT)"
+    fi
+    if sudo nft list chain ip filter LIBVIRT_FWO &>/dev/null 2>&1; then
+        fwd_chain=""  # use individual libvirt chains
+        echo "  using libvirt forward chains (LIBVIRT_FWO/FWI/FWX)"
+    fi
+    if sudo nft list chain ip nat LIBVIRT_PRT &>/dev/null 2>&1; then
+        nat_chain="LIBVIRT_PRT"
+        echo "  using libvirt NAT chain (LIBVIRT_PRT)"
+    fi
+
+    # Check if rules already exist (simple heuristic: look for bridge name in ruleset)
+    local existing
+    existing=$(sudo nft list ruleset 2>/dev/null || true)
+    if echo "$existing" | grep -q "iifname \"$BRIDGE\".*udp dport 67.*accept"; then
+        echo "  firewall rules for $BRIDGE already present, skipping"
+        return 0
+    fi
+
+    echo "  adding INPUT/OUTPUT rules for DHCP and DNS"
+    run_cmd sudo nft add rule ip filter "$inp_chain" iifname "$BRIDGE" udp dport 67 counter accept
+    run_cmd sudo nft add rule ip filter "$inp_chain" iifname "$BRIDGE" udp dport 53 counter accept
+    run_cmd sudo nft add rule ip filter "$inp_chain" iifname "$BRIDGE" tcp dport 53 counter accept
+    run_cmd sudo nft add rule ip filter "$out_chain" oifname "$BRIDGE" udp dport 68 counter accept
+    run_cmd sudo nft add rule ip filter "$out_chain" oifname "$BRIDGE" udp dport 53 counter accept
+
+    echo "  adding FORWARD rules for $net_cidr"
+    if [[ -z "$fwd_chain" ]]; then
+        # libvirt forward chains
+        run_cmd sudo nft add rule ip filter LIBVIRT_FWO ip saddr "$net_cidr" iifname "$BRIDGE" counter accept
+        run_cmd sudo nft add rule ip filter LIBVIRT_FWI ip daddr "$net_cidr" oifname "$BRIDGE" ct state related,established counter accept
+        run_cmd sudo nft add rule ip filter LIBVIRT_FWX iifname "$BRIDGE" oifname "$BRIDGE" counter accept
+    else
+        run_cmd sudo nft add rule ip filter FORWARD ip saddr "$net_cidr" iifname "$BRIDGE" counter accept
+        run_cmd sudo nft add rule ip filter FORWARD ip daddr "$net_cidr" oifname "$BRIDGE" ct state related,established counter accept
+        run_cmd sudo nft add rule ip filter FORWARD iifname "$BRIDGE" oifname "$BRIDGE" counter accept
+    fi
+
+    echo "  adding NAT masquerade for $net_cidr"
+    run_cmd sudo nft add rule ip nat "$nat_chain" ip saddr "$net_cidr" ip daddr 224.0.0.0/24 counter return
+    run_cmd sudo nft add rule ip nat "$nat_chain" ip saddr "$net_cidr" ip daddr 255.255.255.255 counter return
+    run_cmd sudo nft add rule ip nat "$nat_chain" ip saddr "$net_cidr" ip daddr != "$net_cidr" counter masquerade
+
+    echo "  firewall rules configured"
 }
 
 # --- Main ---
@@ -557,6 +742,8 @@ cmd_check() {
     check_bridge_conf
     check_bridge_interface
     check_dhcp
+    check_dhcp_notify
+    check_dhcp_firewall
     check_ip_forward
     check_nat_rules
     check_forward_rules
