feat: start of work to support dra

We want to test if DRA (and dranet) can better
utilize the infiniband devices. We have them
working with UCX, and actually I am not sure
if this will add anything, but it was worth trying
for minimally the learning.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Author: vsoch

Dockerfile.dra

@@ -0,0 +1,17 @@
+FROM ghcr.io/google/dranet:v0.4.0 AS builder
+
+# podman build --userns-uid-map=0:0:1 --userns-uid-map=1:1:1999 --userns-uid-map=65534:2000:2 -f $(pwd)/Dockerfile.dra -t usernetes_dra $(pwd)
+# docker build -t ghcr.io/converged-computing/usernetes:dra .
+# make up && make kubeadm-init && make dra && make kubeconfig && kubectl apply -f Makefile.d/dra/rbac.yaml && podman restart dranet-driver
+
+# We can likely use ubuntu:24.04, I am reproducing the node environment as much as possible.
+FROM docker.io/kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c091b899b2078506e4a0f797fbf6e397c0b0b2
+RUN apt-get update && apt-get install -y ca-certificates
+COPY cspca.llnl.gov.cer.pem /usr/local/share/ca-certificates/
+COPY cspca.cer.pem /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends kmod libibverbs-dev librdmacm-dev rdma-core libnl-3-dev ibverbs-utils libnl-route-3-dev 
+
+COPY --from=builder /dranet /dranet
+ENTRYPOINT ["/dranet"]

Makefile

@@ -84,6 +84,7 @@ up: check-preflight
 	# Podman creates cni files in a shared location, this ensures unique names that do not clobbed one another
 	sed -i "s/default_network/$(HOSTNAME)/g" docker-compose.yaml
 	$(COMPOSE) up --build -d
+	$(NODE_SHELL) /bin/bash /usernetes/Makefile.d/dra/update-containerd.sh
 
 .PHONY: down
 down:
@@ -111,6 +112,15 @@ ifeq ($(shell command -v kubectl 2> /dev/null),)
 	@echo "make kubectl"
 endif
 
+.PHONY: kubeconfig
+dra:
+	$(NODE_SHELL) kubectl apply -f /usernetes/Makefile.d/dra/rbac.yaml
+	# These are needed if you want ibv_devinfo to work
+	#$(CONTAINER_ENGINE) exec dranet-driver mkdir -p /etc/libibverbs.d
+	#$(CONTAINER_ENGINE) exec dranet-driver echo "driver mlx4" >> /etc/libibverbs.d/mlx4.driver
+	#$(CONTAINER_ENGINE) exec dranet-driver echo "driver mlx5" >> /etc/libibverbs.d/mlx5.driver
+
+
 .PHONY: kubectl
 kubectl:
 	$(COMPOSE) exec -T --workdir=/usr/bin $(NODE_SERVICE_NAME) tar c kubectl | tar xv

Makefile.d/dra/rbac.yaml

@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: dranet-admin-role
+rules:
+- apiGroups: ["resource.k8s.io"]
+  resources: ["resourceslices", "resourceclaims", "resourceclaimtemplates"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: dranet-admin-binding
+subjects:
+- kind: User
+  name: kubernetes-admin
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: dranet-admin-role
+  apiGroup: rbac.authorization.k8s.io

Makefile.d/dra/update-containerd.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+if grep -q "io.containerd.nri.v1.nri" /etc/containerd/config.toml
+  then
+    echo "containerd config contains NRI reference already; taking no action"
+  else
+    echo "containerd config does not mention NRI, thus enabling it";
+    printf '%s\n' "[plugins.\"io.containerd.nri.v1.nri\"]" "  disable = false" "  disable_connections = false" "  plugin_config_path = \"/etc/nri/conf.d\"" "  plugin_path = \"/opt/nri/plugins\"" "  plugin_registration_timeout = \"5s\"" "  plugin_request_timeout = \"5s\"" "  socket_path = \"/var/run/nri/nri.sock\"" >> /etc/containerd/config.toml
+    echo "restarting containerd"
+    systemctl restart containerd
+fi

docker-compose.yaml

@@ -10,6 +10,8 @@ services:
     networks:
       default_network:
         ipv4_address: ${NODE_IP}
+    expose:
+      - "${PORT_KUBE_APISERVER}"
     ports:
       # <host>:<container>
       # etcd (default: 2379)
@@ -27,8 +29,7 @@ services:
       - node-var:/var
       - node-opt:/opt
       - node-etc:/etc
-      - type: tmpfs
-        target: /run
+      - node-run:/run
       - type: tmpfs
         target: /tmp
     working_dir: /usernetes
@@ -45,6 +46,29 @@ services:
       "nerdctl/bypass4netns": "${BYPASS4NETNS:-false}"
       "nerdctl/bypass4netns-ignore-bind": "true"
       "nerdctl/bypass4netns-ignore-subnets": "${BYPASS4NETNS_IGNORE_SUBNETS:-}"
+
+  dranet:
+    image: usernetes_dra
+    imagePullPolicy: Always
+    container_name: dranet-driver
+    privileged: true
+    restart: always
+    # DraNet needs to see the host network to manage its devices
+    network_mode: "host"
+    volumes:
+      - /boot:/boot:ro
+      - /lib/modules:/lib/modules:ro
+      - node-var:/var
+      - node-run:/run:ro
+      - node-etc:/etc
+    extra_hosts:
+      # We need to be able to see the api server
+      - "${NODE_NAME}:${HOST_IP}"
+    command:
+      - --kubeconfig=/etc/kubernetes/admin.conf
+      - --hostname-override=${NODE_NAME}
+      - --filter=true 
+
 networks:
   default_network:
     ipam:
@@ -53,6 +77,7 @@ networks:
         # The node IP here is not accessible from other nodes.
         - subnet: ${NODE_SUBNET}
 volumes:
+  node-run: {}
   node-var: {}
   node-opt: {}
   node-etc: {}

kubeadm-config.yaml

@@ -15,6 +15,8 @@ apiServer:
   extraArgs:
   - name: etcd-servers
     value: https://127.0.0.1:${PORT_ETCD}
+  - name: runtime-config
+    value: "api/beta=true"
   - name: advertise-address
     value: ${HOST_IP}
   - name: secure-port
@@ -42,6 +44,8 @@ apiVersion: kubelet.config.k8s.io/v1beta1
 failSwapOn: false
 port: ${PORT_KUBELET}
 featureGates:
+  DRAResourceClaimDeviceStatus: true
+  DynamicResourceAllocation: true
   KubeletInUserNamespace: true
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
@@ -54,3 +58,4 @@ conntrack:
   tcpEstablishedTimeout: 0s
   # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
   tcpCloseWaitTimeout: 0s
+

service/usernetes-start-control-plane.sh

@@ -5,14 +5,24 @@ set -euo pipefail
 # These are variables we likely will change
 # LC only supplies podman
 USERNETES_CONTAINER_TECH=${1:-"podman"} 
-USERNETES_TEMPLATE_PATH=/usr/workspace/usernetes/usernetes-06-26-2025
+USERNETES_TEMPLATE_PATH=/usr/workspace/usernetes/usernetes-dra
 
 # We will copy join command here
 shared_join_command_dir="/usr/workspace/usernetes"
 
 # The user needs to run the setup script
 USERNAME=$(whoami)
 
+# Logging functions for consistency (like Akihiro!)
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - INFO - $1"
+}
+
+error_exit() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - ERROR - $1" >&2
+    exit 1
+}
+
 # This is way a lot for just deriving home, but I'm not convinced it will always
 # be defined in the environment
 if [[ -z "${HOME:-}" || ! -d "${HOME}" ]]; then
@@ -37,16 +47,6 @@ which podman-compose
 # We don't want to use /var because that is a memory based fs
 export TMPDIR="/tmp/${USERNAME}"
 
-# Logging functions for consistency (like Akihiro!)
-log() {
-    echo "$(date '+%Y-%m-%d %H:%M:%S') - INFO - $1"
-}
-
-error_exit() {
-    echo "$(date '+%Y-%m-%d %H:%M:%S') - ERROR - $1" >&2
-    exit 1
-}
-
 install_kubectl() {
     if ! command -v kubectl > /dev/null; then
         log "Installing kubectl..."
@@ -131,6 +131,7 @@ sleep 3 # Allow filesystem operations to settle if needed
 
 log "👷 Building Usernetes container image 'usernetes_node'"
 ${container_runtime_path} build --userns-uid-map=0:0:1 --userns-uid-map=1:1:1999 --userns-uid-map=65534:2000:2 -f $(pwd)/Dockerfile -t usernetes_node $(pwd)
+${container_runtime_path} build --userns-uid-map=0:0:1 --userns-uid-map=1:1:1999 --userns-uid-map=65534:2000:2 -f $(pwd)/Dockerfile.dra -t usernetes_dra $(pwd)
 
 cleanup() {
     log "🧹 Cleaning up old networks or volumes (best effort)"
@@ -150,12 +151,19 @@ if ! make up; then
 fi
 sleep 3
 
+
 log "🔐 Running kubeadm-init with 'make kubeadm-init'"
 if ! make kubeadm-init; then
     error_exit "Failed 'make kubeadm-init'."
 fi
 sleep 3
 
+log "👾 Setting up dynamic resource allocation"
+if ! make dra; then
+    error_exit "Failed 'make kubeadm-init'."
+fi
+sleep 3
+
 log "🥷 Creating kubeconfig with 'make kubeconfig'"
 if ! make kubeconfig; then
     error_exit "Failed 'make kubeconfig'."
@@ -171,6 +179,10 @@ chmod 600 "${KUBECONFIG}"
 # source <(kubectl completion bash)
 sleep 3
 
+log "🥷 Install rbac for dranet-driver"
+kubectl apply -f Makefile.d/dra/rbac.yaml
+${container_runtime_path} restart dranet-driver 
+
 # Get control plane node name robustly
 log "🍑 Untainting control plane and labeling node"
 control_plane_node=""

service/usernetes-start-worker.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # These are variables we likely will change
 # LC only supplies podman
 USERNETES_CONTAINER_TECH=${1:-"podman"} 
-USERNETES_TEMPLATE_PATH=/usr/workspace/usernetes/usernetes-06-26-2025
+USERNETES_TEMPLATE_PATH=/usr/workspace/usernetes/usernetes-dra
 
 # The join command needs to be here
 shared_join_command_dir="/usr/workspace/usernetes"
@@ -17,6 +17,16 @@ fi
 # The user needs to run the setup script
 USERNAME=$(whoami)
 
+# Logging functions for consistency (like Akihiro!)
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - INFO - $1"
+}
+
+error_exit() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - ERROR - $1" >&2
+    exit 1
+}
+
 # This is way a lot for just deriving home, but I'm not convinced it will always
 # be defined in the environment
 if [[ -z "${HOME:-}" || ! -d "${HOME}" ]]; then
@@ -38,16 +48,6 @@ log "    Updated PATH: ${PATH}"
 # We don't want to use /var because that is a memory based fs
 export TMPDIR="/tmp/${USERNAME}"
 
-# Logging functions for consistency (like Akihiro!)
-log() {
-    echo "$(date '+%Y-%m-%d %H:%M:%S') - INFO - $1"
-}
-
-error_exit() {
-    echo "$(date '+%Y-%m-%d %H:%M:%S') - ERROR - $1" >&2
-    exit 1
-}
-
 install_kubectl() {
     if ! command -v kubectl > /dev/null; then
         log "Installing kubectl..."
@@ -135,6 +135,7 @@ sleep 3
 
 log "👷 Building Usernetes container image 'usernetes_node'"
 ${container_runtime_path} build --userns-uid-map=0:0:1 --userns-uid-map=1:1:1999 --userns-uid-map=65534:2000:2 -f $(pwd)/Dockerfile -t usernetes_node $(pwd)
+${container_runtime_path} build --userns-uid-map=0:0:1 --userns-uid-map=1:1:1999 --userns-uid-map=65534:2000:2 -f $(pwd)/Dockerfile.dra -t usernetes_dra $(pwd)
 
 cleanup() {
     log "🧹 Cleaning up old networks or volumes (best effort)"