docker as non-root

Author: Oseltamivir

benchmarks/benchmark_lib.sh

@@ -2,6 +2,53 @@
 
 # Shared benchmarking utilities for InferenceMAX
 
+# ------------------------------
+# Non-root helpers
+# ------------------------------
+
+# Ensure pip and caches target user-writable locations when running as non-root
+_ensure_user_env() {
+    export HOME=${HOME:-/workspace}
+    export XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
+    export PIP_CACHE_DIR=${PIP_CACHE_DIR:-$HOME/.cache/pip}
+    export PYTHONUSERBASE=${PYTHONUSERBASE:-$HOME/.local}
+    # Prepend user base bin so "python3 -m ..." finds scripts if needed
+    case ":$PATH:" in
+        *":$HOME/.local/bin:"*) ;;
+        *) export PATH="$HOME/.local/bin:$PATH" ;;
+    esac
+    mkdir -p "$HOME" "$XDG_CACHE_HOME" "$PIP_CACHE_DIR" "$HOME/.local/bin" || true
+}
+
+# Wrapper for user installs that works without root
+_pip_user_install() {
+    _ensure_user_env
+    python3 -m pip install --user -q --no-cache-dir "$@" || true
+}
+
+# Patch flashinfer cubin_loader in a user overlay instead of system site
+_patch_flashinfer_user_overlay() {
+    set +x
+    _ensure_user_env
+    local src dst parent
+    src=$(python3 - <<'PY'
+import os, flashinfer
+print(os.path.dirname(flashinfer.__file__))
+PY
+    )
+    parent=$(mktemp -d /tmp/fi_patch-XXXXXX)
+    dst="$parent/flashinfer"
+    # Copy package tree to a writable overlay
+    cp -a "$src" "$dst"
+    # Apply the minimal patch
+    if [ -f "$dst/jit/cubin_loader.py" ]; then
+        sed -i '102,108d' "$dst/jit/cubin_loader.py" || true
+    fi
+    # Prepend overlay to PYTHONPATH so it takes precedence even with PYTHONNOUSERSITE=1
+    export PYTHONPATH="$parent:${PYTHONPATH:-}"
+    set -x
+}
+
 # Wait for server to be ready by polling the health endpoint
 # All parameters are required
 # Parameters:
@@ -159,10 +206,10 @@ run_benchmark_serving() {
 # ------------------------------
 
 _install_lm_eval_deps() {
-    python3 -m pip install -q --no-cache-dir "lm-eval[api]" || true
+    _pip_user_install "lm-eval[api]"
     # Temporary: workaround issue by using main
-    python3 -m pip install -q --no-cache-dir --no-deps \
-        "git+https://github.com/EleutherAI/lm-evaluation-harness.git@main" || true
+    _pip_user_install --no-deps \
+        "git+https://github.com/EleutherAI/lm-evaluation-harness.git@main"
 }
 
 # Patch lm-eval filters to be robust to empty strings via sitecustomize
@@ -243,7 +290,9 @@ run_lm_eval() {
     local port="${PORT:-8888}"
     local task="${EVAL_TASK:-gsm8k}"
     local num_fewshot="${NUM_FEWSHOT:-5}"
-    local results_dir="${EVAL_RESULT_DIR:-$(mktemp -d /tmp/eval_out-XXXXXX)}"
+    # Prefer a stable, workspace-mounted location so the host can upload artifacts.
+    # If EVAL_RESULT_DIR is not set, default to /workspace/eval_out/${RESULT_FILENAME}.
+    local results_dir="${EVAL_RESULT_DIR:-/workspace/eval_out/${RESULT_FILENAME:-eval_out}}"
     local gen_max_tokens=4096
     local temperature=0
     local top_p=1
@@ -285,9 +334,9 @@ run_lm_eval() {
 }
 
 append_lm_eval_summary() {
-    local results_dir="${EVAL_RESULT_DIR}"
     local task="${EVAL_TASK:-gsm8k}"
-    local out_dir="${results_dir}"
+    local out_dir="${EVAL_RESULT_DIR:-/workspace/eval_out/${RESULT_FILENAME:-eval_out}}"
+    local results_dir="${out_dir}"
     local summary_md="${out_dir}/SUMMARY.md"
     mkdir -p "$out_dir" || true
 
@@ -326,8 +375,6 @@ META
         fi
     fi
 
-    # Note: Per policy, eval outputs stay under /tmp only; do not copy to workspace.
-
     echo "Results saved to: ${summary_md}"
 }
 
@@ -336,7 +383,7 @@ META
 # ------------------------------
 
 _install_lighteval_deps() {
-    python3 -m pip install -q --no-cache-dir "lighteval[api]" "litellm" || true
+    _pip_user_install "lighteval" "litellm"
 }
 
 # Patch lighteval's LiteLLMClient to handle reasoning content and Python name mangling
@@ -565,7 +612,8 @@ run_lighteval_eval() {
     local port="${PORT:-8888}"
     local task="${EVAL_TASK:-gsm8k}"
     local num_fewshot="${NUM_FEWSHOT:-5}"
-    local results_dir="${EVAL_RESULT_DIR:-eval_out_lighteval}"
+    # Align output path to workspace when not explicitly set
+    local results_dir="${EVAL_RESULT_DIR:-/workspace/eval_out/${RESULT_FILENAME:-eval_out_lighteval}}"
     local max_samples=0
     local concurrent_requests=32
 

benchmarks/dsr1_fp4_b200_docker.sh

@@ -14,9 +14,9 @@
 
 nvidia-smi
 
-# To improve CI stability, we patch this helper function to prevent a race condition that
-# happens 1% of the time. ref: https://github.com/flashinfer-ai/flashinfer/pull/1779
-sed -i '102,108d' /usr/local/lib/python3.12/dist-packages/flashinfer/jit/cubin_loader.py
+# Load helpers and patch flashinfer in a user-writable overlay
+source "$(dirname "$0")/benchmark_lib.sh"
+_patch_flashinfer_user_overlay
 
 SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
 
@@ -40,13 +40,12 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL --host 0.
 
 SERVER_PID=$!
 
-# Source benchmark utilities
-source "$(dirname "$0")/benchmark_lib.sh"
+# Source benchmark utilities (already sourced above)
 
 # Wait for server to be ready
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
-pip install -q datasets pandas
+_pip_user_install datasets pandas
 
 run_benchmark_serving \
     --model "$MODEL" \
@@ -65,4 +64,4 @@ if [ "${RUN_EVAL}" = "true" ]; then
     run_eval --framework lm-eval --port "$PORT" --concurrent-requests $(( $CONC * 2 ))
     append_lm_eval_summary
 fi
-set +x
+set +x

benchmarks/dsr1_fp8_b200_docker.sh

@@ -14,9 +14,9 @@
 
 nvidia-smi
 
-# To improve CI stability, we patch this helper function to prevent a race condition that
-# happens 1% of the time. ref: https://github.com/flashinfer-ai/flashinfer/pull/1779
-sed -i '102,108d' /usr/local/lib/python3.12/dist-packages/flashinfer/jit/cubin_loader.py
+# Load helpers and patch flashinfer in a user-writable overlay
+source "$(dirname "$0")/benchmark_lib.sh"
+_patch_flashinfer_user_overlay
 
 export SGL_ENABLE_JIT_DEEPGEMM=false
 export SGLANG_ENABLE_FLASHINFER_GEMM=true
@@ -42,13 +42,12 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.
 
 SERVER_PID=$!
 
-# Source benchmark utilities
-source "$(dirname "$0")/benchmark_lib.sh"
+# Source benchmark utilities (already sourced above)
 
 # Wait for server to be ready
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
-pip install -q datasets pandas
+_pip_user_install datasets pandas
 
 run_benchmark_serving \
     --model "$MODEL" \

benchmarks/gptoss_fp4_b200_docker.sh

@@ -14,9 +14,9 @@
 
 nvidia-smi
 
-# To improve CI stability, we patch this helper function to prevent a race condition that
-# happens 1% of the time. ref: https://github.com/flashinfer-ai/flashinfer/pull/1779
-sed -i '102,108d' /usr/local/lib/python3.12/dist-packages/flashinfer/jit/cubin_loader.py
+# Load helpers and patch flashinfer in a user-writable overlay
+source "$(dirname "$0")/benchmark_lib.sh"
+_patch_flashinfer_user_overlay
 
 
 # Calculate max-model-len based on ISL and OSL
@@ -56,13 +56,12 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \
 
 SERVER_PID=$!
 
-# Source benchmark utilities
-source "$(dirname "$0")/benchmark_lib.sh"
+# Source benchmark utilities (already sourced above)
 
 # Wait for server to be ready
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
-pip install -q datasets pandas
+_pip_user_install datasets pandas
 
 run_benchmark_serving \
     --model "$MODEL_NAME" \

benchmarks/gptoss_fp4_h100_docker.sh

@@ -43,7 +43,8 @@ source "$(dirname "$0")/benchmark_lib.sh"
 # Wait for server to be ready
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
-pip install -q datasets pandas
+source "$(dirname "$0")/benchmark_lib.sh"
+_pip_user_install datasets pandas
 
 run_benchmark_serving \
     --model "$MODEL_NAME" \

runners/launch_b200-dgxc.sh

@@ -35,13 +35,16 @@ else
 fi
 
 docker run --rm --init --network host --name $server_name \
+--user $(id -u):$(id -g) \
 --runtime nvidia --gpus all --ipc host --privileged --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 \
 -v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 -v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \
 -e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e PORT=$PORT -e EP_SIZE -e DP_ATTENTION \
 -e NCCL_GRAPH_REGISTER=0 \
 -e TORCH_CUDA_ARCH_LIST="10.0" -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" \
 -e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e NUM_PROMPTS \
+-e HOME=/workspace -e XDG_CACHE_HOME=/workspace/.cache -e PIP_CACHE_DIR=/workspace/.cache/pip -e PYTHONUSERBASE=/workspace/.local \
+-e PATH=/workspace/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
 --entrypoint=/bin/bash \
 $(echo "$IMAGE" | sed 's/#/\//') \
 benchmarks/"${EXP_NAME%%_*}_${PRECISION}_b200${FRAMEWORK_SUFFIX}_docker.sh"

runners/launch_b200-nvd.sh

@@ -36,13 +36,16 @@ else
 fi
 
 docker run --rm --init --network host --name $server_name \
+--user $(id -u):$(id -g) \
 --runtime nvidia --gpus all --ipc host --privileged --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 \
 -v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 -v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \
 -e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e PORT=$PORT -e EP_SIZE -e DP_ATTENTION \
 -e NCCL_GRAPH_REGISTER=0 \
 -e TORCH_CUDA_ARCH_LIST="10.0" -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" \
 -e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e NUM_PROMPTS \
+-e HOME=/workspace -e XDG_CACHE_HOME=/workspace/.cache -e PIP_CACHE_DIR=/workspace/.cache/pip -e PYTHONUSERBASE=/workspace/.local \
+-e PATH=/workspace/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
 --entrypoint=/bin/bash \
 $(echo "$IMAGE" | sed 's/#/\//') \
 benchmarks/"${EXP_NAME%%_*}_${PRECISION}_b200${FRAMEWORK_SUFFIX}_docker.sh"

runners/launch_h100-cr.sh

@@ -7,12 +7,15 @@ server_name="bmk-server"
 
 set -x
 docker run --rm --network=host --name=$server_name \
+--user $(id -u):$(id -g) \
 --runtime=nvidia --gpus=all --ipc=host --privileged --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 \
 -v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 -v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \
 -e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e RUN_EVAL -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e PORT=$PORT \
 -e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e TORCH_CUDA_ARCH_LIST="9.0" -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" \
- ${GH_SUM_ENV} ${GH_SUM_MOUNT} \
+${GH_SUM_ENV} ${GH_SUM_MOUNT} \
+-e HOME=/workspace -e XDG_CACHE_HOME=/workspace/.cache -e PIP_CACHE_DIR=/workspace/.cache/pip -e PYTHONUSERBASE=/workspace/.local \
+-e PATH=/workspace/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
 --entrypoint=/bin/bash \
 $IMAGE \
 benchmarks/"${EXP_NAME%%_*}_${PRECISION}_h100_docker.sh"

runners/launch_mi300x-amd.sh

@@ -9,12 +9,15 @@ server_name="bmk-server"
 
 set -x
 docker run --rm --ipc=host --shm-size=16g --network=host --name=$server_name \
+--user $(id -u):$(id -g) \
 --privileged --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri --device=/dev/mem \
 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
 -v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 -v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \
 -e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e CONC -e MAX_MODEL_LEN -e PORT=$PORT \
 -e ISL -e OSL -e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e RANDOM_RANGE_RATIO -e RESULT_FILENAME -e RUN_EVAL \
+ -e HOME=/workspace -e XDG_CACHE_HOME=/workspace/.cache -e PIP_CACHE_DIR=/workspace/.cache/pip -e PYTHONUSERBASE=/workspace/.local \
+ -e PATH=/workspace/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
 ${GH_SUM_ENV} ${GH_SUM_MOUNT} \
 --entrypoint=/bin/bash \
 $IMAGE \

runners/launch_mi300x-cr.sh

@@ -9,12 +9,15 @@ server_name="bmk-server"
 
 set -x
 docker run --rm --ipc=host --shm-size=16g --network=host --name=$server_name \
+--user $(id -u):$(id -g) \
 --privileged --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri --device=/dev/mem \
 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
 -v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 -v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \
 -e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e CONC -e MAX_MODEL_LEN -e PORT=$PORT \
 -e ISL -e OSL -e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e RANDOM_RANGE_RATIO -e RESULT_FILENAME -e RUN_EVAL \
+ -e HOME=/workspace -e XDG_CACHE_HOME=/workspace/.cache -e PIP_CACHE_DIR=/workspace/.cache/pip -e PYTHONUSERBASE=/workspace/.local \
+ -e PATH=/workspace/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
  ${GH_SUM_ENV} ${GH_SUM_MOUNT} \
 --entrypoint=/bin/bash \
 $IMAGE \