|
| 1 | +#!/bin/bash |
| 2 | +# Regression test for the cuDNN packaging in scripts/build/package-gpu-libs.sh. |
| 3 | +# |
| 4 | +# cuDNN 9 is a dispatcher (libcudnn.so.9) plus seven sublibraries the dispatcher |
| 5 | +# dlopen()s by bare soname at runtime. Only the dispatcher is ever a DT_NEEDED, |
| 6 | +# so ldd finds it but never the seven - they have to be completed explicitly. |
| 7 | +# |
| 8 | +# Three end states are correct, and which one applies is a property of the |
| 9 | +# backend, not of the Dockerfile that built it: |
| 10 | +# |
| 11 | +# venv has a complete pip cuDNN -> bundle nothing (longcat-video) |
| 12 | +# venv has no pip cuDNN -> bundle all 8 (vllm: Jetson-index torch |
| 13 | +# links cuDNN, no wheel) |
| 14 | +# no venv, nothing links cuDNN -> bundle nothing (llama-cpp, whisper, |
| 15 | +# rfdetr-cpp, sam3-cpp, |
| 16 | +# stablediffusion-ggml) |
| 17 | +# no venv, something links cuDNN -> bundle all 8 (face-detect, voice-detect) |
| 18 | +# |
| 19 | +# Everything else is a bug this file exists to catch. Historically the allowlist |
| 20 | +# force-copied three cuDNN libs into every CUDA backend, which produced the two |
| 21 | +# failures behind issue #10905: a partial bundle shadowing a complete pip set |
| 22 | +# via LD_LIBRARY_PATH (longcat), and a partial bundle silently completed from |
| 23 | +# the runtime image's system cuDNN (vllm and five Go/C++ backends). |
| 24 | +# |
| 25 | +# Requires gcc (present in the build images); skips otherwise. |
| 26 | +set -euo pipefail |
| 27 | + |
| 28 | +CURDIR=$(dirname "$(realpath "$0")") |
| 29 | +SCRIPT="$CURDIR/package-gpu-libs.sh" |
| 30 | + |
| 31 | +if ! command -v gcc >/dev/null 2>&1; then |
| 32 | + echo "SKIP: gcc not available" |
| 33 | + exit 0 |
| 34 | +fi |
| 35 | + |
| 36 | +WORK=$(mktemp -d) |
| 37 | +trap 'rm -rf "$WORK"' EXIT |
| 38 | + |
| 39 | +# The full cuDNN 9 family as shipped by the libcudnn9-cuda-13 apt package. |
| 40 | +CUDNN_FAMILY=( |
| 41 | + libcudnn |
| 42 | + libcudnn_adv |
| 43 | + libcudnn_cnn |
| 44 | + libcudnn_engines_precompiled |
| 45 | + libcudnn_engines_runtime_compiled |
| 46 | + libcudnn_graph |
| 47 | + libcudnn_heuristic |
| 48 | + libcudnn_ops |
| 49 | +) |
| 50 | + |
| 51 | +echo 'int cudnn_stub(void){return 0;}' > "$WORK/stub.c" |
| 52 | + |
| 53 | +# A consumer must actually CALL into cuDNN, not merely name it on the link line: |
| 54 | +# the toolchain defaults to --as-needed and drops the DT_NEEDED otherwise, which |
| 55 | +# would leave the fixture silently testing nothing. |
| 56 | +printf 'int cudnn_stub(void);\nint consume(void){return cudnn_stub();}\n' > "$WORK/consumer.c" |
| 57 | + |
| 58 | +# A system (apt) cuDNN: real .so.9.24.0 files behind .so.9 symlinks. This is |
| 59 | +# what the L4T build image carries and it holds no TensorRT, matching reality - |
| 60 | +# nothing in the Dockerfiles installs libnvinfer. |
| 61 | +SYS="$WORK/sys" |
| 62 | +mkdir -p "$SYS" |
| 63 | +for name in "${CUDNN_FAMILY[@]}"; do |
| 64 | + gcc -shared -fPIC -o "$SYS/${name}.so.9.24.0" "$WORK/stub.c" |
| 65 | + ln -s "${name}.so.9.24.0" "$SYS/${name}.so.9" |
| 66 | +done |
| 67 | + |
| 68 | +# Same, plus a TensorRT stand-in that DT_NEEDEDs cuDNN. Used to prove a bundled |
| 69 | +# library pulling cuDNN in also gets the family completed, and that excluding |
| 70 | +# cuDNN never over-excludes its dependents. |
| 71 | +SYSTRT="$WORK/systrt" |
| 72 | +mkdir -p "$SYSTRT" |
| 73 | +cp -a "$SYS"/. "$SYSTRT/" |
| 74 | +gcc -shared -fPIC -o "$SYSTRT/libnvinfer.so.10" "$WORK/consumer.c" \ |
| 75 | + -L"$SYS" -l:libcudnn.so.9 -Wl,-rpath,"$SYS" |
| 76 | + |
| 77 | +# Build a backend dir: <edir>/lib is the bundle target, <edir>/venv is the venv. |
| 78 | +# pip ships cuDNN as plain libcudnn*.so.9 files with no versioned real name. |
| 79 | +# |
| 80 | +# "links-cudnn" reproduces the Go/C++ layout: package.sh stages the backend's |
| 81 | +# own shared object into package/lib, which IS the target dir, so the existing |
| 82 | +# transitive sweep sees it. That is how face-detect/voice-detect are detected. |
| 83 | +# |
| 84 | +# $1 = backend name, $2 = pip-cudnn | venv-no-cudnn | no-venv | links-cudnn |
| 85 | +make_backend() { |
| 86 | + local edir="$WORK/$1" |
| 87 | + mkdir -p "$edir/lib" |
| 88 | + case "$2" in |
| 89 | + pip-cudnn) |
| 90 | + local sp="$edir/venv/lib/python3.12/site-packages/nvidia/cudnn/lib" |
| 91 | + mkdir -p "$sp" |
| 92 | + local name |
| 93 | + for name in "${CUDNN_FAMILY[@]}"; do |
| 94 | + gcc -shared -fPIC -o "$sp/${name}.so.9" "$WORK/stub.c" |
| 95 | + done |
| 96 | + ;; |
| 97 | + venv-no-cudnn) |
| 98 | + mkdir -p "$edir/venv/lib/python3.12/site-packages/nvidia" |
| 99 | + ;; |
| 100 | + links-cudnn) |
| 101 | + gcc -shared -fPIC -o "$edir/lib/libfacedetect.so" "$WORK/consumer.c" \ |
| 102 | + -L"$SYS" -l:libcudnn.so.9 -Wl,-rpath,"$SYS" |
| 103 | + ;; |
| 104 | + no-venv) ;; |
| 105 | + esac |
| 106 | + echo "$edir" |
| 107 | +} |
| 108 | + |
| 109 | +# Run the packager for one backend in a fresh bash. A ( ) subshell would inherit |
| 110 | +# the COPIED_FILES dedup map from a previous run and skip everything. |
| 111 | +# $1 = backend lib dir, $2 = system lib dir |
| 112 | +run_packager() { |
| 113 | + env BUILD_TYPE=l4t CUDA_LIB_DIRS="$2" \ |
| 114 | + bash -c 'source "$1" "$2"; package_cuda_libs' _ "$SCRIPT" "$1" 2>&1 |
| 115 | +} |
| 116 | + |
| 117 | +bundled_cudnn() { |
| 118 | + find "$1" -maxdepth 1 -name 'libcudnn*' -printf '%f\n' 2>/dev/null | sort | tr '\n' ' ' |
| 119 | +} |
| 120 | + |
| 121 | +# Report how many of the 8 sonames are present, for the "expect all" cases. |
| 122 | +missing_cudnn() { |
| 123 | + local dir="$1" name out=() |
| 124 | + for name in "${CUDNN_FAMILY[@]}"; do |
| 125 | + [ -e "$dir/${name}.so.9" ] || out+=("${name}.so.9") |
| 126 | + done |
| 127 | + echo "${out[*]:-}" |
| 128 | +} |
| 129 | + |
| 130 | +rc=0 |
| 131 | +pass() { echo "PASS: $1"; } |
| 132 | +fail() { echo "FAIL: $1"; rc=1; } |
| 133 | + |
| 134 | +# --- 1. venv provides a complete pip cuDNN (longcat-video). |
| 135 | +EDIR=$(make_backend pipbackend pip-cudnn) |
| 136 | +run_packager "$EDIR/lib" "$SYSTRT" >/dev/null 2>&1 || true |
| 137 | +leaked=$(bundled_cudnn "$EDIR/lib") |
| 138 | +if [ -z "$leaked" ]; then |
| 139 | + pass "venv with complete pip cuDNN -> nothing bundled" |
| 140 | +else |
| 141 | + fail "venv already has cuDNN but we bundled: $leaked" |
| 142 | +fi |
| 143 | +if [ -e "$EDIR/lib/libnvinfer.so.10" ]; then |
| 144 | + pass "excluding cuDNN does not over-exclude its dependents" |
| 145 | +else |
| 146 | + fail "excluding cuDNN dropped libnvinfer.so.10 too" |
| 147 | +fi |
| 148 | + |
| 149 | +# --- 2. venv exists but ships NO pip cuDNN (vllm). The consumers live inside |
| 150 | +# the venv where the sweep cannot see them, so this stays conservative. |
| 151 | +EDIR=$(make_backend vllmbackend venv-no-cudnn) |
| 152 | +run_packager "$EDIR/lib" "$SYS" >/dev/null 2>&1 || true |
| 153 | +missing=$(missing_cudnn "$EDIR/lib") |
| 154 | +if [ -z "$missing" ]; then |
| 155 | + pass "venv without pip cuDNN -> complete family bundled" |
| 156 | +else |
| 157 | + fail "venv without pip cuDNN left the backend short of cuDNN: $missing" |
| 158 | +fi |
| 159 | + |
| 160 | +# --- 3. no venv and nothing links cuDNN (llama-cpp, whisper, rfdetr-cpp, |
| 161 | +# sam3-cpp, stablediffusion-ggml). ggml uses cuBLAS, not cuDNN. These carry |
| 162 | +# ~57 MB of partial cuDNN today; completing the family for them would take that |
| 163 | +# to ~576 MB, all of it for libraries with no consumer. |
| 164 | +EDIR=$(make_backend gonocudnn no-venv) |
| 165 | +run_packager "$EDIR/lib" "$SYS" >/dev/null 2>&1 || true |
| 166 | +leaked=$(bundled_cudnn "$EDIR/lib") |
| 167 | +if [ -z "$leaked" ]; then |
| 168 | + pass "no venv and nothing links cuDNN -> nothing bundled" |
| 169 | +else |
| 170 | + fail "bundled cuDNN into a backend with no cuDNN consumer: $leaked" |
| 171 | +fi |
| 172 | + |
| 173 | +# --- 4. no venv but the backend's own .so links cuDNN (face-detect, |
| 174 | +# voice-detect, built with -DFACEDETECT_GGML_CUDNN=ON on arm64 + CUDA 13). |
| 175 | +EDIR=$(make_backend golinkscudnn links-cudnn) |
| 176 | +run_packager "$EDIR/lib" "$SYS" >/dev/null 2>&1 || true |
| 177 | +missing=$(missing_cudnn "$EDIR/lib") |
| 178 | +if [ -z "$missing" ]; then |
| 179 | + pass "no venv but the backend links cuDNN -> complete family bundled" |
| 180 | +else |
| 181 | + fail "backend links cuDNN but the family was left incomplete: $missing" |
| 182 | +fi |
| 183 | + |
| 184 | +# --- 5. a bundled library pulling cuDNN in must also get the family completed. |
| 185 | +# Nothing installs TensorRT today, but if it ever is, libnvinfer's DT_NEEDED on |
| 186 | +# libcudnn would otherwise reintroduce exactly the partial set from #10905. |
| 187 | +EDIR=$(make_backend gotrt no-venv) |
| 188 | +run_packager "$EDIR/lib" "$SYSTRT" >/dev/null 2>&1 || true |
| 189 | +missing=$(missing_cudnn "$EDIR/lib") |
| 190 | +if [ -z "$missing" ]; then |
| 191 | + pass "a dependent dragging cuDNN in -> complete family bundled" |
| 192 | +else |
| 193 | + fail "libnvinfer pulled cuDNN in but the family was left incomplete: $missing" |
| 194 | +fi |
| 195 | + |
| 196 | +# --- 6. no cuDNN in the build image at all (every non-arm64 CUDA image). |
| 197 | +EDIR=$(make_backend nocudnnanywhere venv-no-cudnn) |
| 198 | +NOSYS="$WORK/nosys" |
| 199 | +mkdir -p "$NOSYS" |
| 200 | +if run_packager "$EDIR/lib" "$NOSYS" >/dev/null 2>&1; then |
| 201 | + pass "no cuDNN in the build image and none needed -> build still succeeds" |
| 202 | +else |
| 203 | + fail "build failed for a backend that has no cuDNN available anywhere" |
| 204 | +fi |
| 205 | + |
| 206 | +# --- Guard unit checks. Source once for direct access to the helpers. |
| 207 | +mkdir -p "$WORK/guard" |
| 208 | +# shellcheck source=/dev/null |
| 209 | +source "$SCRIPT" "$WORK/guard" |
| 210 | + |
| 211 | +for fn in verify_cudnn_bundle cudnn_family_state cudnn_venv_lib_dir cudnn_is_referenced; do |
| 212 | + if ! declare -F "$fn" >/dev/null; then |
| 213 | + echo "FAIL: package-gpu-libs.sh does not define $fn" |
| 214 | + exit 1 |
| 215 | + fi |
| 216 | +done |
| 217 | + |
| 218 | +# The shape five Go/C++ backends plus vllm ship today: three of eight bundled, |
| 219 | +# no venv cuDNN, a complete cuDNN in the build image. It survives only because |
| 220 | +# the runtime image's system cuDNN completes the family - libcudnn_cnn.so.9 has |
| 221 | +# a hard DT_NEEDED on libcudnn_graph.so.9, which none of them bundle, so it |
| 222 | +# resolves to /lib/aarch64-linux-gnu and the process runs bundled 9.22.0 against |
| 223 | +# system 9.23.2. The build image is not the runtime image; this must not pass. |
| 224 | +FLEET="$WORK/fleetshape" |
| 225 | +mkdir -p "$FLEET" |
| 226 | +for name in libcudnn libcudnn_cnn libcudnn_ops; do |
| 227 | + cp "$SYS/${name}.so.9.24.0" "$FLEET/${name}.so.9.22.0" |
| 228 | + ln -s "${name}.so.9.22.0" "$FLEET/${name}.so.9" |
| 229 | +done |
| 230 | +if verify_cudnn_bundle "$FLEET" absent complete 2>/dev/null; then |
| 231 | + fail "verify_cudnn_bundle accepted the venv=0 bundled=3 fleet shape" |
| 232 | +else |
| 233 | + pass "verify_cudnn_bundle rejected the venv=0 bundled=3 fleet shape" |
| 234 | +fi |
| 235 | + |
| 236 | +# Mixed-version bundle: two cuDNN builds in one directory. |
| 237 | +MIXED="$WORK/mixed" |
| 238 | +mkdir -p "$MIXED" |
| 239 | +for name in "${CUDNN_FAMILY[@]}"; do |
| 240 | + cp "$SYS/${name}.so.9.24.0" "$MIXED/" |
| 241 | + ln -sf "${name}.so.9.24.0" "$MIXED/${name}.so.9" |
| 242 | +done |
| 243 | +cp "$SYS/libcudnn_adv.so.9.24.0" "$MIXED/libcudnn_adv.so.9.20.0" |
| 244 | +ln -sf libcudnn_adv.so.9.20.0 "$MIXED/libcudnn_adv.so.9" |
| 245 | +if verify_cudnn_bundle "$MIXED" absent absent 2>/dev/null; then |
| 246 | + fail "verify_cudnn_bundle accepted a mixed 9.24.0 / 9.20.0 bundle" |
| 247 | +else |
| 248 | + pass "verify_cudnn_bundle rejected a mixed-version bundle" |
| 249 | +fi |
| 250 | + |
| 251 | +# cuDNN in BOTH the bundle and the venv: the bundle shadows the venv, so even |
| 252 | +# two individually complete sets are a misconfiguration. |
| 253 | +COMPLETE="$WORK/complete" |
| 254 | +mkdir -p "$COMPLETE" |
| 255 | +for name in "${CUDNN_FAMILY[@]}"; do |
| 256 | + cp "$SYS/${name}.so.9.24.0" "$COMPLETE/" |
| 257 | + ln -sf "${name}.so.9.24.0" "$COMPLETE/${name}.so.9" |
| 258 | +done |
| 259 | +if verify_cudnn_bundle "$COMPLETE" complete absent 2>/dev/null; then |
| 260 | + fail "verify_cudnn_bundle accepted cuDNN in both the bundle and the venv" |
| 261 | +else |
| 262 | + pass "verify_cudnn_bundle rejected cuDNN in both the bundle and the venv" |
| 263 | +fi |
| 264 | + |
| 265 | +# Zero cuDNN is CORRECT when nothing references it - that is llama-cpp, whisper |
| 266 | +# and friends, and it is the common case. The guard must not demand a cuDNN for |
| 267 | +# backends that never call one just because the build image happens to have it. |
| 268 | +EMPTY="$WORK/empty" |
| 269 | +mkdir -p "$EMPTY" |
| 270 | +if verify_cudnn_bundle "$EMPTY" absent complete; then |
| 271 | + pass "zero cuDNN accepted when nothing references it" |
| 272 | +else |
| 273 | + fail "verify_cudnn_bundle demanded a cuDNN no consumer asked for" |
| 274 | +fi |
| 275 | + |
| 276 | +# Zero cuDNN is WRONG when something does reference it. A backend whose own .so |
| 277 | +# links the dispatcher and that ends up with no cuDNN cannot load at all. |
| 278 | +NEEDS="$WORK/needscudnn" |
| 279 | +mkdir -p "$NEEDS" |
| 280 | +gcc -shared -fPIC -o "$NEEDS/libfacedetect.so" "$WORK/consumer.c" \ |
| 281 | + -L"$SYS" -l:libcudnn.so.9 -Wl,-rpath,"$SYS" |
| 282 | +if verify_cudnn_bundle "$NEEDS" absent complete 2>/dev/null; then |
| 283 | + fail "verify_cudnn_bundle accepted zero cuDNN for a backend that links it" |
| 284 | +else |
| 285 | + pass "verify_cudnn_bundle rejected zero cuDNN for a backend that links it" |
| 286 | +fi |
| 287 | + |
| 288 | +# cudnn_is_referenced must see a dlopen()ed soname too, not just DT_NEEDED. |
| 289 | +# A consumer that only ever dlopen()s cuDNN has the string in .rodata and no |
| 290 | +# dynamic entry at all, so an ldd-based check would miss it entirely. |
| 291 | +DLOPEN="$WORK/dlopenonly" |
| 292 | +mkdir -p "$DLOPEN" |
| 293 | +printf 'const char *n="libcudnn.so.9";\nint f(void){return 0;}\n' > "$WORK/dl.c" |
| 294 | +gcc -shared -fPIC -o "$DLOPEN/libdlopener.so" "$WORK/dl.c" |
| 295 | +if cudnn_is_referenced "$DLOPEN"; then |
| 296 | + pass "cudnn_is_referenced detects a dlopen-only consumer" |
| 297 | +else |
| 298 | + fail "cudnn_is_referenced missed a dlopen-only consumer (ldd cannot see it)" |
| 299 | +fi |
| 300 | + |
| 301 | +# The venv-only end state stays valid. |
| 302 | +if verify_cudnn_bundle "$EMPTY" complete complete; then |
| 303 | + pass "verify_cudnn_bundle accepts the venv-only end state" |
| 304 | +else |
| 305 | + fail "verify_cudnn_bundle rejected the correct venv-only end state" |
| 306 | +fi |
| 307 | + |
| 308 | +exit $rc |
0 commit comments