Add support for multiple interfaces (#102)

* Add support for multiple interfaces

* Broadcast speed should be set per interface

Author: eliasol

.github/workflows/rust.yml

@@ -47,6 +47,7 @@ jobs:
           sudo apt-get install -y dnsmasq qemu-system-x86 ovmf
           sudo mkdir -p /etc/qemu
           echo allow br-pixie | sudo tee /etc/qemu/bridge.conf
+          echo allow br-pixie1 | sudo tee -a /etc/qemu/bridge.conf
 
       - name: Install rust
         uses: actions-rust-lang/setup-rust-toolchain@v1

pixie-server/example.config.yaml

@@ -1,9 +1,10 @@
 hosts:
-  listen_on: 10.0.0.1
-  dhcp: !static [10.187.100.1, 10.187.200.200]
-  #dhcp: !proxy 192.168.1.100
+  interfaces:
+  - network: 10.0.0.1/8
+    dhcp: !static [10.187.100.1, 10.187.200.200]
+    #dhcp: !proxy 192.168.1.100
+    broadcast_speed: 52428800
   hostsfile: /etc/hosts
-  broadcast_speed: 52428800
 http:
   listen_on: 0.0.0.0:8080
   #password: secret

pixie-server/src/dnsmasq.rs

@@ -35,29 +35,48 @@ impl Drop for DnsmasqHandle {
 }
 
 async fn write_config(state: &State) -> Result<()> {
-    let (name, _) = find_network(state.config.hosts.listen_on)?;
-
     let mut dnsmasq_conf = File::create(state.run_dir.join("dnsmasq.conf"))?;
 
-    let dhcp_dynamic_conf = match state.config.hosts.dhcp {
-        DhcpMode::Static(low, high) => format!("dhcp-range=tag:netboot,{low},{high}"),
-        DhcpMode::Proxy(ip) => format!("dhcp-range=tag:netboot,{ip},proxy"),
-    };
-
     let storage_str = state.storage_dir.to_str().unwrap();
     let run_str = state.run_dir.to_str().unwrap();
 
+    let interfaces_config = state
+        .config
+        .hosts
+        .interfaces
+        .iter()
+        .map(|iface| {
+            let name = find_network(iface.network.addr())?.0;
+
+            let dhcp_dynamic_conf = match iface.dhcp {
+                DhcpMode::Static(low, high) => format!("dhcp-range=tag:netboot,{low},{high}"),
+                DhcpMode::Proxy(ip) => format!("dhcp-range=tag:netboot,{ip},proxy"),
+            };
+
+            let netaddr = iface.network.network().to_string();
+            let netmask = iface.network.netmask().to_string();
+
+            Ok(format!(
+                r#"
+## {name}
+dhcp-range=tag:!netboot,{netaddr},static,{netmask}
+{dhcp_dynamic_conf}
+interface={name}
+"#
+            ))
+        })
+        .collect::<Result<Vec<_>>>()?
+        .join("\n");
+
     write!(
         dnsmasq_conf,
         r#"
 ### Per-network configuration
 
-## net0
-{dhcp_dynamic_conf}
-dhcp-range=tag:!netboot,10.0.0.0,static,255.0.0.0
+{interfaces_config}
+
 dhcp-hostsfile={run_str}/hosts
 dhcp-boot=pixie-uefi.efi
-interface={name}
 except-interface=lo
 user=root
 group=root

pixie-server/src/udp.rs

@@ -1,17 +1,19 @@
 //! Handles [`UdpRequest`]
 
 use crate::{
-    find_mac, find_network,
+    find_mac,
     state::{State, UnitSelector},
 };
 use anyhow::{ensure, Context, Result};
+use futures::FutureExt;
+use ipnet::Ipv4Net;
 use pixie_shared::{
-    chunk_codec::Encoder, ChunkHash, HintPacket, RegistrationInfo, UdpRequest, ACTION_PORT,
-    CHUNKS_PORT, HINT_PORT, UDP_BODY_LEN,
+    chunk_codec::Encoder, ChunkHash, HintPacket, InterfaceConfig, RegistrationInfo, UdpRequest,
+    ACTION_PORT, CHUNKS_PORT, HINT_PORT, UDP_BODY_LEN,
 };
 use std::{
     collections::BTreeSet,
-    net::{Ipv4Addr, SocketAddrV4},
+    net::{IpAddr, Ipv4Addr, SocketAddrV4},
     ops::Bound,
     sync::Arc,
 };
@@ -24,7 +26,7 @@ use tokio::{
 async fn broadcast_chunks(
     state: &State,
     socket: &UdpSocket,
-    ip: Ipv4Addr,
+    iface: &InterfaceConfig,
     mut rx: Receiver<ChunkHash>,
 ) -> Result<()> {
     let mut queue = BTreeSet::<ChunkHash>::new();
@@ -75,8 +77,7 @@ async fn broadcast_chunks(
             continue;
         };
 
-        let hosts_cfg = &state.config.hosts;
-        let chunks_addr = SocketAddrV4::new(ip, CHUNKS_PORT);
+        let chunks_addr = SocketAddrV4::new(iface.network.broadcast(), CHUNKS_PORT);
 
         let mut encoder = Encoder::new(cdata);
         write_buf[..32].clone_from_slice(&index);
@@ -85,7 +86,7 @@ async fn broadcast_chunks(
 
             let sent_len = socket.send_to(&write_buf[..32 + len], chunks_addr).await?;
             ensure!(sent_len == 32 + len, "Could not send packet");
-            wait_for += 8 * (sent_len as u32) * Duration::from_secs(1) / hosts_cfg.broadcast_speed;
+            wait_for += 8 * (sent_len as u32) * Duration::from_secs(1) / iface.broadcast_speed;
         }
     }
 
@@ -165,13 +166,24 @@ async fn broadcast_hint(state: &State, socket: &UdpSocket, ip: Ipv4Addr) -> Resu
     Ok(())
 }
 
-async fn handle_requests(state: &State, socket: &UdpSocket, tx: Sender<[u8; 32]>) -> Result<()> {
+async fn handle_requests(
+    state: &State,
+    socket: &UdpSocket,
+    net_tx: Vec<(Ipv4Net, Sender<[u8; 32]>)>,
+) -> Result<()> {
     let mut buf = [0; UDP_BODY_LEN];
     loop {
         let (len, peer_addr) = tokio::select! {
             x = socket.recv_from(&mut buf) => x?,
             _ = state.cancel_token.cancelled() => break,
         };
+        let peer_ip = match peer_addr.ip() {
+            IpAddr::V4(ip) => ip,
+            _ => panic!(),
+        };
+        let Some((_, tx)) = net_tx.iter().find(|(net, _)| net.contains(&peer_ip)) else {
+            continue;
+        };
         let req: postcard::Result<UdpRequest> = postcard::from_bytes(&buf[..len]);
         match req {
             Ok(UdpRequest::Discover) => {
@@ -201,18 +213,29 @@ async fn handle_requests(state: &State, socket: &UdpSocket, tx: Sender<[u8; 32]>
 }
 
 pub async fn main(state: Arc<State>) -> Result<()> {
-    let (_, network) = find_network(state.config.hosts.listen_on)?;
+    let (net_tx, net_rx): (_, Vec<_>) = state
+        .config
+        .hosts
+        .interfaces
+        .iter()
+        .map(|iface| {
+            let (tx, rx) = mpsc::channel(128);
+            ((iface.network, tx), (iface, rx))
+        })
+        .unzip();
 
-    let (tx, rx) = mpsc::channel(128);
     let socket = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, ACTION_PORT)).await?;
     log::info!("Listening on {}", socket.local_addr()?);
     socket.set_broadcast(true)?;
 
-    tokio::try_join!(
-        broadcast_chunks(&state, &socket, network.broadcast(), rx),
-        broadcast_hint(&state, &socket, network.broadcast()),
-        handle_requests(&state, &socket, tx),
-    )?;
+    let mut tasks = vec![handle_requests(&state, &socket, net_tx).boxed()];
+
+    for (iface, rx) in net_rx {
+        tasks.push(broadcast_chunks(&state, &socket, iface, rx).boxed());
+        tasks.push(broadcast_hint(&state, &socket, iface.network.broadcast()).boxed());
+    }
+
+    futures::future::try_join_all(tasks).await?;
 
     Ok(())
 }

pixie-shared/src/config.rs

@@ -1,5 +1,6 @@
 use crate::{Action, Bijection};
 use alloc::{string::String, vec::Vec};
+use ipnet::Ipv4Net;
 use macaddr::MacAddr6;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -18,20 +19,26 @@ pub enum DhcpMode {
     Proxy(Ipv4Addr),
 }
 
+#[derive(Debug, Eq, PartialEq, Serialize, Deserialize, Clone)]
+pub struct InterfaceConfig {
+    /// Listen on address
+    pub network: Ipv4Net,
+    /// DHCP server.
+    pub dhcp: DhcpMode,
+    /// Speed in bytes/second used to broadcast chunks.
+    pub broadcast_speed: u32,
+}
+
 /// Registered clients will always be assigned an IP in the form
 /// 10.{group_id}.{column_id}.{row_id}.
 /// Note that for this to work, the specified network interface must have an IP on the 10.0.0.0/8
 /// subnet; BEWARE that dnsmasq can be picky about the order of IP addresses.
 #[derive(Debug, Eq, PartialEq, Serialize, Deserialize, Clone)]
 pub struct HostsConfig {
-    /// Listen on address
-    pub listen_on: Ipv4Addr,
-    /// DHCP server.
-    pub dhcp: DhcpMode,
+    /// Interfaces to operate on.
+    pub interfaces: Vec<InterfaceConfig>,
     /// Hosts file to use for DHCP hostnames.
     pub hostsfile: Option<PathBuf>,
-    /// Speed in bytes/second used to broadcast chunks.
-    pub broadcast_speed: u32,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]

run_test.sh

@@ -21,6 +21,7 @@ cleanup() {
 trap cleanup EXIT
 
 cp -rv $1 $TEMPDIR
+cp test_config.yaml $TEMPDIR/storage/config.yaml
 
 cat >$TEMPDIR/storage/registered.json <<EOF
 [
@@ -33,14 +34,30 @@ cat >$TEMPDIR/storage/registered.json <<EOF
     "curr_progress": null,
     "next_action": "shutdown",
     "image": "contestant"
+  },
+  {
+    "mac": [82, 84, 0, 18, 52, 87],
+    "group": 10,
+    "row": 9,
+    "col": 1,
+    "curr_action": null,
+    "curr_progress": null,
+    "next_action": "shutdown",
+    "image": "contestant"
   }
 ]
 EOF
 
 if ! ip link show br-pixie; then
   ip link add br-pixie type bridge
   ip link set dev br-pixie up
-  ip addr add 10.0.0.1/8 brd 10.255.255.255 dev br-pixie
+  ip addr add 10.0.0.1/16 dev br-pixie
+fi
+
+if ! ip link show br-pixie1; then
+  ip link add br-pixie1 type bridge
+  ip link set dev br-pixie1 up
+  ip addr add 10.10.0.1/16 dev br-pixie1
 fi
 
 RUST_BACKTRACE=short RUST_LOG=debug RUST_LOG_STYLE=always LLVM_PROFILE_FILE=prof-out/pixie-server-%m-%p.profraw ./pixie-server/target/debug/pixie-server -s $TEMPDIR/storage &
@@ -67,7 +84,30 @@ run_qemu() {
     -nic bridge,mac=52:54:00:12:34:56,br=br-pixie,model=e1000
 }
 
+run_qemu1() {
+  OVMF=/usr/share/OVMF/OVMF_CODE_4M.fd
+  if ! [ -e $OVMF ]; then
+    OVMF=/usr/share/edk2/x64/OVMF_CODE.4m.fd
+  fi
+  FILE=prof-out/pixie-uefi-$RANDOM.profraw
+  truncate -s 500M $FILE
+  qemu-system-x86_64 \
+    -nographic \
+    -chardev stdio,id=char0,logfile=$1,signal=off \
+    -serial chardev:char0 \
+    -monitor none \
+    -enable-kvm \
+    -cpu host -smp cores=2 \
+    -m 1G \
+    -drive if=pflash,format=raw,file=$OVMF \
+    -drive file=$TEMPDIR/disk1.img,if=none,id=nvm,format=raw \
+    -drive file=$FILE,format=raw \
+    -device nvme,serial=deadbeef,drive=nvm \
+    -nic bridge,mac=52:54:00:12:34:57,br=br-pixie1,model=e1000
+}
+
 truncate -s 8G $TEMPDIR/disk.img
+truncate -s 8G $TEMPDIR/disk1.img
 echo -e "label: gpt\n- 1GiB - -\n- 1GiB - -\n- - L -" | sfdisk $TEMPDIR/disk.img
 DEV=$(losetup --partscan --show --find $TEMPDIR/disk.img)
 mkswap ${DEV}p1
@@ -80,7 +120,7 @@ for PART in ${DEV}p{2..3}; do
 done
 losetup -d $DEV
 
-curl 'http://localhost:8080/admin/curr_action/all/store'
+curl 'http://localhost:8080/admin/curr_action/52:54:00:12:34:56/store'
 run_qemu $TEMPDIR/store.log
 
 # Check that we restore the original disk image.
@@ -90,6 +130,7 @@ truncate -s 8G $TEMPDIR/disk.img
 
 curl 'http://localhost:8080/admin/curr_action/all/flash'
 run_qemu $TEMPDIR/flash-1.log
+run_qemu1 $TEMPDIR/flash1-1.log
 
 DEV=$(losetup --partscan --show --find --read-only $TEMPDIR/disk.img)
 fsck -n ${DEV}p*
@@ -103,10 +144,23 @@ for PART in ${DEV}p{2..3}; do
 done
 losetup -d $DEV
 
+DEV=$(losetup --partscan --show --find --read-only $TEMPDIR/disk1.img)
+fsck -n ${DEV}p*
+for PART in ${DEV}p{2..3}; do
+  mount -o ro $PART $TEMPDIR/mnt
+  if [ "$(md5sum $TEMPDIR/mnt/pixie-server | cut -f 1 -d ' ')" != "$(md5sum ./pixie-server/target/debug/pixie-server | cut -f 1 -d ' ')" ]; then
+    echo "pixie-server does not contain the expected content"
+    exit 1
+  fi
+  umount $TEMPDIR/mnt
+done
+losetup -d $DEV
+
 # Check that we don't fetch any data if the disk contents have not changed.
 
 curl 'http://localhost:8080/admin/curr_action/all/flash'
 run_qemu $TEMPDIR/flash-2.log
+run_qemu1 $TEMPDIR/flash1-2.log
 
 DEV=$(losetup --partscan --show --find --read-only $TEMPDIR/disk.img)
 fsck -n ${DEV}p*
@@ -120,7 +174,24 @@ for PART in ${DEV}p{2..3}; do
 done
 losetup -d $DEV
 
+DEV=$(losetup --partscan --show --find --read-only $TEMPDIR/disk1.img)
+fsck -n ${DEV}p*
+for PART in ${DEV}p{2..3}; do
+  mount -o ro $PART $TEMPDIR/mnt
+  if [ "$(md5sum $TEMPDIR/mnt/pixie-server | cut -f 1 -d ' ')" != "$(md5sum ./pixie-server/target/debug/pixie-server | cut -f 1 -d ' ')" ]; then
+    echo "pixie-server does not contain the expected content"
+    exit 1
+  fi
+  umount $TEMPDIR/mnt
+done
+losetup -d $DEV
+
 if ! grep "Disk scanned; 0 chunks to fetch" $TEMPDIR/flash-2.log &>/dev/null; then
   echo "Data was re-fetched"
   exit 1
 fi
+
+if ! grep "Disk scanned; 0 chunks to fetch" $TEMPDIR/flash1-2.log &>/dev/null; then
+  echo "Data was re-fetched"
+  exit 1
+fi

test_config.yaml

@@ -0,0 +1,17 @@
+hosts:
+  interfaces:
+  - network: 10.0.0.1/16
+    dhcp: !static [10.0.100.1, 10.0.200.200]
+    broadcast_speed: 52428800
+  - network: 10.10.0.1/16
+    dhcp: !static [10.10.100.1, 10.10.200.200]
+    broadcast_speed: 52428800
+  hostsfile: /etc/hosts
+http:
+  listen_on: 0.0.0.0:8080
+groups:
+  - [room0, 0]
+  - [room10, 10]
+images:
+  - contestant
+  - worker