update

vickiegpt · vickiegpt · commit 004b6384a838 · 2026-06-12T18:08:56.000Z
diff --git a/qemu_integration/README.md b/qemu_integration/README.md
@@ -170,6 +170,25 @@ sudo CXL_REGION_TYPE=ram CXL_DAX_MODE=devdax ./setup_cxl_numa.sh
 sudo CXL_REGION_TYPE=ram CXL_DAX_MODE=system-ram ./setup_cxl_numa.sh
 ```
 
+When installing `cxl-numa-setup.service` in a guest, put per-VM settings in
+`/etc/default/cxl-numa-setup`. The setup helper derives a default static IP as
+`192.168.100.(10 + CXL_HOST_ID)/24` when `CXL_CONFIGURE_NET=1` and
+`CXL_NET_ADDR` is not set:
+
+```bash
+# Primary VM
+CXL_CONFIGURE_NET=1
+CXL_HOST_ID=0
+
+# Secondary VM
+CXL_CONFIGURE_NET=1
+CXL_HOST_ID=1
+```
+
+Set `CXL_NET_ADDR` explicitly if the VM should use a different address. The
+helper auto-detects the first non-loopback network interface by default; set
+`CXL_NET_IFACE=enp0s2` or similar to pin it.
+
 For DCD/volatile CXL.mem, use `daxctl`/device-dax or system-ram mode. The old
 `ndctl create-namespace -m dax` path is for pmem-style regions and is disabled
 by default in `setup_cxl_numa.sh`; enable it only with
diff --git a/qemu_integration/cxl-numa-setup.service b/qemu_integration/cxl-numa-setup.service
@@ -8,10 +8,11 @@ DefaultDependencies=no
 [Service]
 Type=oneshot
 RemainAfterExit=yes
+EnvironmentFile=-/etc/default/cxl-numa-setup
 ExecStart=/usr/local/bin/setup_cxl_numa.sh
 StandardOutput=journal
 StandardError=journal
 TimeoutSec=60
 
 [Install]
-WantedBy=sysinit.target
+WantedBy=sysinit.target
diff --git a/qemu_integration/launch_qemu_cxl_usernet.sh b/qemu_integration/launch_qemu_cxl_usernet.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euo pipefail
+
+QEMU_BINARY=${QEMU_BINARY:-/usr/local/bin/qemu-system-x86_64}
+ROOT=${ROOT:-/home/victoryang00/CXLMemSim}
+KERNEL=${KERNEL:-$ROOT/build/bzImage}
+DISK_IMAGE=${DISK_IMAGE:-$ROOT/build/qemu.img}
+VM_MEMORY=${VM_MEMORY:-8G}
+VM_MAX_MEMORY=${VM_MAX_MEMORY:-16G}
+SMP=${SMP:-4}
+SSH_PORT=${SSH_PORT:-10022}
+CXL_BACKING=${CXL_BACKING:-/dev/shm/cxlmemsim_shared}
+CXL_LSA=${CXL_LSA:-/dev/shm/lsa0.raw}
+CXL_BACKING_SIZE=${CXL_BACKING_SIZE:-1G}
+CXL_LSA_SIZE=${CXL_LSA_SIZE:-256M}
+CXL_FMW_SIZE=${CXL_FMW_SIZE:-4G}
+
+truncate -s "$CXL_BACKING_SIZE" "$CXL_BACKING"
+truncate -s "$CXL_LSA_SIZE" "$CXL_LSA"
+
+export CXL_TRANSPORT_MODE=${CXL_TRANSPORT_MODE:-shm}
+export CXL_PGAS_SHM=${CXL_PGAS_SHM:-/cxlmemsim_pgas}
+export CXL_HOST_ID=${CXL_HOST_ID:-0}
+export CXL_MEMSIM_HOST=${CXL_MEMSIM_HOST:-127.0.0.1}
+export CXL_MEMSIM_PORT=${CXL_MEMSIM_PORT:-9999}
+
+exec "$QEMU_BINARY" \
+    --enable-kvm \
+    -cpu qemu64,+xsave,+rdtscp,+avx,+avx2,+sse4.1,+sse4.2,+clflushopt \
+    -m "$VM_MEMORY",maxmem="$VM_MAX_MEMORY",slots=8 \
+    -smp "$SMP" \
+    -M q35,cxl=on \
+    -kernel "$KERNEL" \
+    -append "root=/dev/sda rw console=ttyS0,115200 nokaslr" \
+    -drive file="$DISK_IMAGE",index=0,media=disk,format=raw \
+    -netdev user,id=net0,hostfwd=tcp:127.0.0.1:"$SSH_PORT"-:22 \
+    -device virtio-net-pci,netdev=net0,mac=52:54:00:00:10:22 \
+    -fsdev local,security_model=none,id=fsdev0,path="$ROOT" \
+    -device virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=hostrepo,bus=pcie.0 \
+    -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
+    -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=0 \
+    -device cxl-type3,bus=root_port13,persistent-memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem0,sn=0x1 \
+    -object memory-backend-file,id=cxl-mem1,share=on,mem-path="$CXL_BACKING",size="$CXL_BACKING_SIZE" \
+    -object memory-backend-file,id=cxl-lsa1,share=on,mem-path="$CXL_LSA",size="$CXL_LSA_SIZE" \
+    -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size="$CXL_FMW_SIZE" \
+    -nographic
diff --git a/qemu_integration/launch_qemu_type2_multigpu.sh b/qemu_integration/launch_qemu_type2_multigpu.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+# Launch QEMU with N CXL Type-2 accelerator contexts attached to one host VM.
+# Used by the Splash multi-GPU scaling sweep.
+#
+# Environment:
+#   NUM_TYPE2        - Number of Type-2 accelerators to attach (1, 2, 4, 6, 8).
+#                      Default: 1.
+#   QEMU_BINARY      - Path to qemu-system-x86_64.
+#   DISK_IMAGE       - Path to the guest rootfs image (relative CWD).
+#   KERNEL_IMAGE     - Path to bzImage.
+#   SSH_PORT         - hostfwd guest:22 -> host:${SSH_PORT}. Default 10022.
+#   HETGPU_BACKEND   - 0 auto / 3 nvidia / 5 simulation. Default 5.
+#   HETGPU_LIB       - Path to libnvcuda.so.
+#   CXL_TYPE2_CACHE_SIZE / CXL_TYPE2_MEM_SIZE - per-device sizes.
+#   DIRECTORY_ENTRIES - snoop filter capacity (0 uses device default).
+#   MONITOR_SOCK     - QEMU monitor unix socket path.
+#   VM_LOG           - File to redirect serial console + stderr.
+
+set -e
+
+NUM_TYPE2=${NUM_TYPE2:-1}
+QEMU_BINARY=${QEMU_BINARY:-/home/victoryang00/CXLMemSim/lib/qemu/build/qemu-system-x86_64}
+DISK_IMAGE=${DISK_IMAGE:-qemu1.img}
+DISK_IMAGE_PATH=${DISK_IMAGE_PATH:-}
+# If no absolute path provided, default to <repo>/build/<DISK_IMAGE> which is
+# the canonical location, independent of the invoker's working directory.
+if [ -z "${DISK_IMAGE_PATH}" ]; then
+    DISK_IMAGE_PATH="/home/victoryang00/CXLMemSim/build/${DISK_IMAGE}"
+fi
+KERNEL_IMAGE=${KERNEL_IMAGE:-/home/victoryang00/cxl/arch/x86/boot/bzImage}
+SSH_PORT=${SSH_PORT:-10022}
+HETGPU_BACKEND=${HETGPU_BACKEND:-5}
+HETGPU_LIB=${HETGPU_LIB:-/home/victoryang00/hetGPU/target/debug/libnvcuda.so}
+CXL_TYPE2_CACHE_SIZE=${CXL_TYPE2_CACHE_SIZE:-128M}
+CXL_TYPE2_MEM_SIZE=${CXL_TYPE2_MEM_SIZE:-4G}
+DIRECTORY_ENTRIES=${DIRECTORY_ENTRIES:-0}
+MONITOR_SOCK=${MONITOR_SOCK:-/tmp/qemu-mon.sock}
+VM_LOG=${VM_LOG:-/home/victoryang00/CXLMemSim/artifact/splash_sweep/vm1.log}
+CXL_MEMSIM_HOST=${CXL_MEMSIM_HOST:-127.0.0.1}
+CXL_MEMSIM_PORT=${CXL_MEMSIM_PORT:-9999}
+
+if ! [[ "$NUM_TYPE2" =~ ^[0-9]+$ ]] || [ "$NUM_TYPE2" -lt 1 ] || [ "$NUM_TYPE2" -gt 8 ]; then
+    echo "NUM_TYPE2 must be an integer between 1 and 8" >&2
+    exit 1
+fi
+
+RP_OPTS=()
+T2_OPTS=()
+for i in $(seq 0 $((NUM_TYPE2-1))); do
+    PORT=$((13 + i))
+    RP_ID="root_port${PORT}"
+    T2_ID="cxl-type2-hetgpu${i}"
+    SN=$(printf "0x%x" $((0x10 + i)))
+    RP_OPTS+=( -device "cxl-rp,port=${i},bus=cxl.1,id=${RP_ID},chassis=0,slot=${i}" )
+    T2_OPTS+=( -device "cxl-type2,bus=${RP_ID},cache-size=${CXL_TYPE2_CACHE_SIZE},mem-size=${CXL_TYPE2_MEM_SIZE},sn=${SN},cxlmemsim-addr=${CXL_MEMSIM_HOST},cxlmemsim-port=${CXL_MEMSIM_PORT},coherency-enabled=true,gpu-mode=2,hetgpu-lib=${HETGPU_LIB},hetgpu-device=0,hetgpu-backend=${HETGPU_BACKEND},directory-entries=${DIRECTORY_ENTRIES},id=${T2_ID}" )
+done
+
+exec "${QEMU_BINARY}" \
+    --enable-kvm -cpu qemu64,+xsave,+rdtscp,+avx,+avx2,+sse4.1,+sse4.2,+avx512f,+avx512dq,+avx512ifma,+avx512cd,+avx512bw,+avx512vl,+avx512vbmi,+clflushopt \
+    -kernel "${KERNEL_IMAGE}" \
+    -append "root=/dev/vda rw console=ttyS0,115200 nokaslr systemd.mask=cxl-numa-setup.service" \
+    -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22" \
+    -device virtio-net-pci,netdev=net0,mac=52:54:00:00:00:02 \
+    -device virtio-rng-pci \
+    -drive file="${DISK_IMAGE_PATH:-./${DISK_IMAGE}}",if=none,id=disk0,format=raw \
+    -device virtio-blk-pci,drive=disk0,bus=pcie.0 \
+    -M q35,cxl=on -m 16G,maxmem=32G,slots=8 -smp 4 \
+    -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
+    "${RP_OPTS[@]}" \
+    "${T2_OPTS[@]}" \
+    -monitor "unix:${MONITOR_SOCK},server,nowait" \
+    -D /dev/null \
+    -nographic > "${VM_LOG}" 2>&1
diff --git a/qemu_integration/setup_cxl_numa.sh b/qemu_integration/setup_cxl_numa.sh
@@ -20,9 +20,21 @@ CXL_DAX_MODE=${CXL_DAX_MODE:-devdax}
 CXL_TOUCH_DAX=${CXL_TOUCH_DAX:-${ZETTAI_TOUCH_DAX:-0}}
 CXL_CREATE_NDCTL_NAMESPACE=${CXL_CREATE_NDCTL_NAMESPACE:-0}
 CXL_CONFIGURE_NET=${CXL_CONFIGURE_NET:-0}
-CXL_NET_IFACE=${CXL_NET_IFACE:-enp0s2}
-CXL_NET_ADDR=${CXL_NET_ADDR:-192.168.100.10/24}
+CXL_HOST_ID=${CXL_HOST_ID:-${ZETTAI_HOST_ID:-0}}
+CXL_NET_IFACE=${CXL_NET_IFACE:-auto}
+CXL_NET_BASE=${CXL_NET_BASE:-192.168.100}
+CXL_NET_PREFIX=${CXL_NET_PREFIX:-24}
+CXL_NET_HOST_OFFSET=${CXL_NET_HOST_OFFSET:-10}
+if [[ -z "${CXL_NET_ADDR:-}" ]]; then
+    if [[ "$CXL_HOST_ID" =~ ^[0-9]+$ && "$CXL_NET_HOST_OFFSET" =~ ^[0-9]+$ ]]; then
+        CXL_NET_ADDR="${CXL_NET_BASE}.$((CXL_NET_HOST_OFFSET + CXL_HOST_ID))/${CXL_NET_PREFIX}"
+    else
+        CXL_NET_ADDR="${CXL_NET_BASE}.${CXL_NET_HOST_OFFSET}/${CXL_NET_PREFIX}"
+    fi
+fi
 CXL_NET_GW=${CXL_NET_GW:-192.168.100.1}
+CXL_NET_WAIT_RETRIES=${CXL_NET_WAIT_RETRIES:-10}
+CXL_NET_WAIT_DELAY=${CXL_NET_WAIT_DELAY:-0.2}
 MAX_RETRIES=${MAX_RETRIES:-20}
 RETRY_DELAY=${RETRY_DELAY:-2}
 
@@ -44,6 +56,9 @@ Environment:
   CXL_TOUCH_DAX                  mmap and touch /dev/daxX.Y, default: 0
   CXL_CREATE_NDCTL_NAMESPACE     Legacy ndctl namespace path, default: 0
   CXL_CONFIGURE_NET              Configure static guest network, default: 0
+  CXL_HOST_ID                    Node id used for default IP, default: 0
+  CXL_NET_IFACE                  Interface name or auto, default: auto
+  CXL_NET_ADDR                   Static address, default: 192.168.100.(10+CXL_HOST_ID)/24
 EOF
 }
 
@@ -304,9 +319,68 @@ configure_network() {
         return 0
     fi
 
-    ip link set "$CXL_NET_IFACE" up
-    ip addr add "$CXL_NET_ADDR" dev "$CXL_NET_IFACE" 2>/dev/null || true
-    ip route add default via "$CXL_NET_GW" 2>/dev/null || true
+    if ! command -v ip >/dev/null 2>&1; then
+        log "WARNING: ip command is not installed; skipping network setup"
+        return 0
+    fi
+
+    local iface=""
+    if ! iface=$(wait_for_net_iface); then
+        log "WARNING: no network interface available for CXL_NET_IFACE=$CXL_NET_IFACE; skipping network setup"
+        return 0
+    fi
+
+    log "Configuring guest network: iface=$iface addr=$CXL_NET_ADDR gw=$CXL_NET_GW"
+    if ! ip link set "$iface" up; then
+        log "WARNING: failed to bring $iface up; skipping network setup"
+        return 0
+    fi
+
+    ip addr add "$CXL_NET_ADDR" dev "$iface" 2>/dev/null || true
+    if [[ -n "$CXL_NET_GW" ]]; then
+        ip route add default via "$CXL_NET_GW" 2>/dev/null || true
+    fi
+}
+
+detect_net_iface() {
+    if [[ "$CXL_NET_IFACE" != "auto" ]]; then
+        if ip link show dev "$CXL_NET_IFACE" >/dev/null 2>&1; then
+            echo "$CXL_NET_IFACE"
+            return 0
+        fi
+        return 1
+    fi
+
+    local iface=""
+    iface=$(ip -o link show up 2>/dev/null |
+        awk -F': ' '$2 != "lo" { sub(/@.*/, "", $2); print $2; exit }')
+    if [[ -z "$iface" ]]; then
+        iface=$(ip -o link show 2>/dev/null |
+            awk -F': ' '$2 != "lo" { sub(/@.*/, "", $2); print $2; exit }')
+    fi
+
+    if [[ -n "$iface" ]]; then
+        echo "$iface"
+        return 0
+    fi
+    return 1
+}
+
+wait_for_net_iface() {
+    local retries=0
+    local iface=""
+
+    while [[ $retries -lt $CXL_NET_WAIT_RETRIES ]]; do
+        if iface=$(detect_net_iface); then
+            echo "$iface"
+            return 0
+        fi
+        log "Waiting for guest network interface... (attempt $((retries + 1))/$CXL_NET_WAIT_RETRIES)"
+        sleep "$CXL_NET_WAIT_DELAY"
+        retries=$((retries + 1))
+    done
+
+    return 1
 }
 
 main() {
diff --git a/qemu_integration/test_setup_cxl_numa.sh b/qemu_integration/test_setup_cxl_numa.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+SCRIPT="$SCRIPT_DIR/setup_cxl_numa.sh"
+TMP_DIR=$(mktemp -d)
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+STUB_DIR="$TMP_DIR/bin"
+STATE_DIR="$TMP_DIR/state"
+mkdir -p "$STUB_DIR" "$STATE_DIR"
+
+cat >"$STUB_DIR/cxl" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+
+case "$*" in
+    "list -M"*)
+        echo '[{"memdev":"mem0"}]'
+        ;;
+    "list -B -D"*)
+        echo '[{"decoder":"decoder0.0","volatile_capable":true,"max_available_extent":268435456}]'
+        ;;
+    "list -R"*)
+        if [[ -f "$CXL_TEST_STATE/region_created" ]]; then
+            echo '[{"region":"region0"}]'
+        else
+            echo '[]'
+        fi
+        ;;
+    create-region*)
+        touch "$CXL_TEST_STATE/region_created"
+        echo '{"region":"region0"}'
+        ;;
+    list*)
+        echo '[]'
+        ;;
+    *)
+        echo "unexpected cxl invocation: $*" >&2
+        exit 2
+        ;;
+esac
+STUB
+
+cat >"$STUB_DIR/daxctl" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+
+case "${1:-}" in
+    list)
+        echo '[{"chardev":"dax0.0"}]'
+        ;;
+    create-device)
+        echo '{"chardev":"dax0.0"}'
+        ;;
+    reconfigure-device)
+        exit 0
+        ;;
+    *)
+        echo "unexpected daxctl invocation: $*" >&2
+        exit 2
+        ;;
+esac
+STUB
+
+cat >"$STUB_DIR/ip" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+
+printf '%s\n' "$*" >>"$CXL_TEST_STATE/ip.calls"
+
+case "$*" in
+    "-o link show up"|"-o link show")
+        echo '2: enp0s2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP mode DEFAULT group default qlen 1000'
+        ;;
+    link\ set\ *|addr\ add\ *|route\ add\ *)
+        exit 0
+        ;;
+    *)
+        echo "unexpected ip invocation: $*" >&2
+        exit 2
+        ;;
+esac
+STUB
+
+cat >"$STUB_DIR/modprobe" <<'STUB'
+#!/usr/bin/env bash
+exit 0
+STUB
+
+cat >"$STUB_DIR/udevadm" <<'STUB'
+#!/usr/bin/env bash
+exit 0
+STUB
+
+cat >"$STUB_DIR/numactl" <<'STUB'
+#!/usr/bin/env bash
+echo 'available: 1 nodes (0)'
+STUB
+
+chmod +x "$STUB_DIR"/*
+
+export PATH="$STUB_DIR:$PATH"
+export CXL_TEST_STATE="$STATE_DIR"
+
+LOG_FILE="$TMP_DIR/cxl_numa_setup.log"
+
+LOG_FILE="$LOG_FILE" \
+MAX_RETRIES=1 \
+RETRY_DELAY=0 \
+CXL_CONFIGURE_NET=1 \
+CXL_HOST_ID=1 \
+bash "$SCRIPT" >"$TMP_DIR/script.stdout" 2>"$TMP_DIR/script.stderr"
+
+if ! grep -q 'addr add 192.168.100.11/24 dev enp0s2' "$STATE_DIR/ip.calls"; then
+    echo "FAIL: secondary host id should default to 192.168.100.11/24" >&2
+    echo "ip calls:" >&2
+    cat "$STATE_DIR/ip.calls" >&2
+    echo "setup log:" >&2
+    cat "$LOG_FILE" >&2
+    exit 1
+fi
+
+echo "OK: setup_cxl_numa secondary network defaults"
diff --git a/script/setup_cxl_numa.sh b/script/setup_cxl_numa.sh
diff --git a/workloads/gromacs/mpi_cxl_shim.c b/workloads/gromacs/mpi_cxl_shim.c
