Merge pull request #35 from dmvevents/add-nvfp4-bf16-toggle

nemotron/ultra: NVFP4/BF16 precision toggle via MODEL_VARIANT (.env, no template change)

Author: iankouls-aws

Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/agg/.env

@@ -13,8 +13,22 @@ export INSTANCE_TYPE_FRONTEND=m5.8xlarge
 export INSTANCE_TYPE_WORKER=p5en.48xlarge
 export GPU_PER_WORKER=8
 export EFA_PER_WORKER=16
-export MODEL_NAME=nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16
-export MODEL_PATH=/shared/models/hf/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16
+# ---- Model precision toggle (Author: Anton Alexander) ----
+# Switch between the full-precision BF16 checkpoint and the NVFP4 (4-bit) checkpoint
+# WITHOUT editing the templates. BF16 is the default; set MODEL_VARIANT=NVFP4 to switch.
+#   - bf16  : nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16  (~1.1 TB weights; needs the full node budget)
+#   - nvfp4 : nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 (4-bit weights; ~41 GiB/GPU, big KV headroom)
+# vLLM auto-detects the quantization from the checkpoint (no --quantization flag needed), so only the
+# model name/path change. Override MODEL_REPO to point at any other HF repo following the same -BF16/-NVFP4
+# suffix convention.
+export MODEL_VARIANT="${MODEL_VARIANT:-BF16}"
+export MODEL_REPO="${MODEL_REPO:-nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B}"
+case "${MODEL_VARIANT}" in
+  nvfp4|NVFP4|fp4) _PREC_SUFFIX="NVFP4" ;;
+  bf16|BF16|*)     _PREC_SUFFIX="BF16"  ;;
+esac
+export MODEL_NAME="${MODEL_REPO}-${_PREC_SUFFIX}"
+export MODEL_PATH="/shared/models/hf/${MODEL_NAME}"
 
 export ETCD_URL=http://dynamo-platform-etcd.dynamo-system.svc.cluster.local:2379
 export NATS_URL=nats://dynamo-platform-nats.dynamo-system.svc.cluster.local:4222

Container-Root/eks/deployment/inference/agentic-ai/nemotron/ultra/disagg/.env

@@ -13,8 +13,22 @@ export INSTANCE_TYPE_FRONTEND=m5.8xlarge
 export INSTANCE_TYPE_WORKER=p5en.48xlarge
 export GPU_PER_WORKER=8
 export EFA_PER_WORKER=16
-export MODEL_NAME=nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16
-export MODEL_PATH=/shared/models/hf/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16
+# ---- Model precision toggle (Author: Anton Alexander) ----
+# Switch between the full-precision BF16 checkpoint and the NVFP4 (4-bit) checkpoint
+# WITHOUT editing the templates. BF16 is the default; set MODEL_VARIANT=NVFP4 to switch.
+#   - bf16  : nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16  (~1.1 TB weights; needs the full node budget)
+#   - nvfp4 : nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 (4-bit weights; ~41 GiB/GPU, big KV headroom)
+# vLLM auto-detects the quantization from the checkpoint (no --quantization flag needed), so only the
+# model name/path change. Override MODEL_REPO to point at any other HF repo following the same -BF16/-NVFP4
+# suffix convention.
+export MODEL_VARIANT="${MODEL_VARIANT:-BF16}"
+export MODEL_REPO="${MODEL_REPO:-nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B}"
+case "${MODEL_VARIANT}" in
+  nvfp4|NVFP4|fp4) _PREC_SUFFIX="NVFP4" ;;
+  bf16|BF16|*)     _PREC_SUFFIX="BF16"  ;;
+esac
+export MODEL_NAME="${MODEL_REPO}-${_PREC_SUFFIX}"
+export MODEL_PATH="/shared/models/hf/${MODEL_NAME}"
 
 export ETCD_URL=http://dynamo-platform-etcd.dynamo-system.svc.cluster.local:2379
 export NATS_URL=nats://dynamo-platform-nats.dynamo-system.svc.cluster.local:4222