Skip to content

Commit 9736c4d

Browse files
mc-nvmc-nv
authored
Update TensorRT-LLM build versions. (#850) (#851)
* Update TensorRT-LLM build versions. (#850) * Update TensorRT-LLM submodule to release/1.2@36e34eeff * Update versions * Update base image versiosn * Update TensorRT-LLM version * uninsall wheel package to pass the patch step * I'm not happy with this commit... * Fixing broken installation * Reinstall broken package * Update docker image * Adding torchgen module, it's vital for torch package * Moving torchgen next to torch they both part of the same package * Reevaluating python objects * Update versions and package installation * Setuptools should come as TensroRT-LLM dependency * Update submodule versions * Wrapping pytorch dependencies * Update submodule * Update submodule * Update submodule commit * UPdate submodule commit
1 parent e8c09ea commit 9736c4d

2 files changed

Lines changed: 92 additions & 107 deletions

File tree

‎dockerfile/Dockerfile.triton.trt_llm_backend‎

Lines changed: 91 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,22 @@
1-
ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.10-py3-min
2-
ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:25.10-py3
3-
ARG NVRTC_VER=13.0.48-1
4-
ARG TRT_VER=10.13.3.9
5-
ARG NCCL_VER=2.27.7-1+cuda13.0
6-
ARG RELEASE_URL_TRT_x86=https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.13.3/tars/TensorRT-${TRT_VER}.Linux.x86_64-gnu.cuda-13.0.tar.gz
7-
ARG RELEASE_URL_TRT_ARM=https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.13.3/tars/TensorRT-${TRT_VER}.Linux.aarch64-gnu.cuda-13.0.tar.gz
1+
ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.12-py3-min
2+
ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:25.12-py3
3+
ARG TRT_VER=10.14.1.48
4+
ARG RELEASE_URL_TRT_x86=https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.14.1/tars/TensorRT-${TRT_VER}.Linux.x86_64-gnu.cuda-13.0.tar.gz
5+
ARG RELEASE_URL_TRT_ARM=https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.14.1/tars/TensorRT-${TRT_VER}.Linux.aarch64-gnu.cuda-13.0.tar.gz
86

97
# Versions of packages to copy from pytorch image
10-
ARG FLASH_ATTN_VER=2.7.4.post1+25.10
11-
ARG NETWORKX_VER=3.5
12-
ARG PACKAGING_VER=25.0
13-
ARG PYTORCH_TRITON_VER=3.4.0+gitc817b9b6
14-
ARG SETUPTOOLS_VER=79.0.1
15-
ARG SYMPY_VER=1.14.0
16-
ARG TORCH_VER=2.9.0a0+145a3a7bda.nv25.10
17-
ARG TORCHVISION_VER=0.24.0a0+094e7af5
8+
ARG PIP_WHL_VER_FLASH_ATTN=2.7.4.post1+25.12
9+
ARG PIP_WHL_VER_NVFUSER=0.2.34+git073e91b
10+
ARG PIP_WHL_VER_PYCOCOTOOLS=2.0+nv0.8.1
11+
ARG PIP_WHL_VER_PYTORCH_TRITON=3.5.1+gitbfeb0668.nv25.12
12+
ARG PIP_WHL_VER_TORCH=2.10.0a0+b4e4ee81d3.nv25.12
13+
ARG PIP_WHL_VER_TORCHAO=0.15.0+git01374eb5
14+
ARG PIP_WHL_VER_TORCHVISION=0.25.0a0+ca221243
15+
ARG PIP_WHL_VER_TRITON_KERNELS=1.0.0+gitbfeb0668.nv25.12
1816

1917
ARG TENSORRTLLM_REPO=https://github.com/NVIDIA/TensorRT-LLM.git
20-
ARG TENSORRTLLM_REPO_TAG=release/1.1
21-
ARG TENSORRTLLM_VER=1.1.0
18+
ARG TENSORRTLLM_REPO_TAG=release/1.2
19+
ARG TENSORRTLLM_VER=1.2.0rc8
2220

2321
FROM ${PYTORCH_IMAGE} AS pytorch_image
2422
FROM ${BASE_IMAGE} AS install_dependencies
@@ -28,43 +26,23 @@ WORKDIR /workspace
2826
ENV PIP_BREAK_SYSTEM_PACKAGES=1
2927
RUN apt-get update -q=2 \
3028
&& apt-get install -y --no-install-recommends \
31-
python3-dev \
32-
python3-pip \
29+
python3.12-minimal \
30+
python3.12-venv \
3331
git-lfs \
3432
# Remove previous TRT installation
3533
&& apt-get purge -y "libnvinfer*" \
36-
&& pip3 uninstall -y tensorrt \
34+
&& python3.12 -m venv /opt/venv-tritonserver \
3735
&& rm -rf /var/lib/apt/lists/*
3836

37+
ENV PATH="/opt/venv-tritonserver/bin:${PATH}"
38+
3939
ARG TRT_VER
40-
ARG NVRTC_VER
41-
ARG NCCL_VER
4240

4341
ENV TRT_VERSION=$TRT_VER \
4442
TRT_VER=$TRT_VER \
45-
CUDA_VER=$CUDA_VERSION \
46-
CUDNN_VER=$CUDNN_VERSION \
47-
NCCL_VER=$NCCL_VER \
48-
CUBLAS_VER=$CUBLAS_VERSION \
49-
NVRTC_VER="${NVRTC_VER}"
43+
CUDA_VER=$CUDA_VERSION
5044

5145
LABEL TRT_VERSION=$TRT_VER
52-
LABEL NCCL_VER=$NCCL_VER
53-
54-
# Install NVRTC
55-
RUN [ "$(uname -m)" != "x86_64" ] && arch="sbsa" || arch="x86_64" \
56-
&& curl -o /tmp/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$arch/cuda-keyring_1.1-1_all.deb \
57-
&& apt install /tmp/cuda-keyring.deb \
58-
&& rm /tmp/cuda-keyring.deb \
59-
&& apt-get remove --purge -y --allow-change-held-packages cuda-nvrtc-dev* \
60-
&& CUDA_VER_SHORT=${CUDA_VER: 0:4} \
61-
&& NVRTC_CUDA_VERSION=${CUDA_VER_SHORT/./-} \
62-
&& apt-get update -qq \
63-
&& apt-get install -y --no-install-recommends \
64-
cuda-nvrtc-dev-${NVRTC_CUDA_VERSION}=${NVRTC_VER} \
65-
libnccl2=${NCCL_VER} \
66-
libnccl-dev=${NCCL_VER} \
67-
&& rm -rf /var/lib/apt/lists/*
6846

6947
# Download & install TRT release
7048
ARG RELEASE_URL_TRT_x86
@@ -93,79 +71,79 @@ RUN git-lfs install \
9371
&& git clone --single-branch --recurse-submodules --depth=1 -b ${TENSORRTLLM_REPO_TAG} ${TENSORRTLLM_REPO} tensorrt_llm
9472

9573
# Final stage to build the TRT-LLM container
96-
FROM ${BASE_IMAGE} AS final_stage
97-
98-
ARG TORCH_VER
99-
ARG TORCHVISION_VER
100-
ARG SETUPTOOLS_VER
101-
ARG PYTORCH_TRITON_VER
102-
ARG NETWORKX_VER
103-
ARG SYMPY_VER
104-
ARG PACKAGING_VER
105-
ARG FLASH_ATTN_VER
106-
# Copy necessary files from the base stage
107-
COPY --from=pytorch_image /usr/local/lib/lib* /usr/local/lib/
108-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torch /usr/local/lib/python3.12/dist-packages/torch
109-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torch-${TORCH_VER}.dist-info /usr/local/lib/python3.12/dist-packages/torch-${TORCH_VER}.dist-info
110-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchgen /usr/local/lib/python3.12/dist-packages/torchgen
111-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchvision /usr/local/lib/python3.12/dist-packages/torchvision
112-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchvision-${TORCHVISION_VER}.dist-info /usr/local/lib/python3.12/dist-packages/torchvision-${TORCHVISION_VER}.dist-info
113-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchvision.libs /usr/local/lib/python3.12/dist-packages/torchvision.libs
114-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/setuptools /usr/local/lib/python3.12/dist-packages/setuptools
115-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/setuptools-${SETUPTOOLS_VER}.dist-info /usr/local/lib/python3.12/dist-packages/setuptools-${SETUPTOOLS_VER}.dist-info
116-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/functorch /usr/local/lib/python3.12/dist-packages/functorch
117-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/pytorch_triton-${PYTORCH_TRITON_VER}.dist-info /usr/local/lib/python3.12/dist-packages/pytorch_triton-${PYTORCH_TRITON_VER}.dist-info
118-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/triton /usr/local/lib/python3.12/dist-packages/triton
119-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/networkx /usr/local/lib/python3.12/dist-packages/networkx
120-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/networkx-${NETWORKX_VER}.dist-info /usr/local/lib/python3.12/dist-packages/networkx-${NETWORKX_VER}.dist-info
121-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/sympy /usr/local/lib/python3.12/dist-packages/sympy
122-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/sympy-${SYMPY_VER}.dist-info /usr/local/lib/python3.12/dist-packages/sympy-${SYMPY_VER}.dist-info
123-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/packaging /usr/local/lib/python3.12/dist-packages/packaging
124-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/packaging-${PACKAGING_VER}.dist-info /usr/local/lib/python3.12/dist-packages/packaging-${PACKAGING_VER}.dist-info
125-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/flash_attn /usr/local/lib/python3.12/dist-packages/flash_attn
126-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/flash_attn-${FLASH_ATTN_VER}.dist-info /usr/local/lib/python3.12/dist-packages/flash_attn-${FLASH_ATTN_VER}.dist-info
127-
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/flash_attn_2_cuda.cpython-312-*-linux-gnu.so /usr/local/lib/python3.12/dist-packages/
128-
129-
ARG NVRTC_VER
130-
ARG NCCL_VER
131-
ENV CUDA_VER=$CUDA_VERSION \
132-
NVRTC_VER="${NVRTC_VER}" \
133-
NCCL_VER="${NCCL_VER}"
74+
FROM ${BASE_IMAGE} AS pytorch_wrapper
13475

13576
# Install the necessary dependencies and remove previous TRT installation in the
13677
# final image
137-
ENV PIP_BREAK_SYSTEM_PACKAGES=1
78+
ARG PIP_WHL_VER_FLASH_ATTN
79+
ARG PIP_WHL_VER_NVFUSER
80+
ARG PIP_WHL_VER_PYCOCOTOOLS
81+
ARG PIP_WHL_VER_PYTORCH_TRITON
82+
ARG PIP_WHL_VER_TORCH
83+
ARG PIP_WHL_VER_TORCHAO
84+
ARG PIP_WHL_VER_TORCHVISION
85+
ARG PIP_WHL_VER_TRANSFORMER_ENGINE
86+
ARG PIP_WHL_VER_TRITON_KERNELS
87+
88+
# Copy necessary files from the base stage python3.12/dist-packages/torch-${PIP_WHL_VER_TORCH}.dist-info
89+
COPY --from=pytorch_image /usr/local/lib/lib* /usr/local/lib/
90+
# FLASH_ATTN
91+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/flash_attn /opt/venv-tritonserver/lib/python3.12/site-packages/flash_attn
92+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/flash_attn-${PIP_WHL_VER_FLASH_ATTN}.dist-info /opt/venv-tritonserver/lib/python3.12/site-packages/flash_attn-${PIP_WHL_VER_FLASH_ATTN}.dist-info
93+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/flash_attn_2_cuda.cpython-312-*-linux-gnu.so /opt/venv-tritonserver/lib/python3.12/site-packages/flash_attn_2_cuda.cpython-312-*-linux-gnu.so
94+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/hopper /opt/venv-tritonserver/lib/python3.12/site-packages/hopper
95+
# NVFUSER
96+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/nvfuser /opt/venv-tritonserver/lib/python3.12/site-packages/nvfuser
97+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/nvfuser-${PIP_WHL_VER_NVFUSER}.dist-info /opt/venv-tritonserver/lib/python3.12/site-packages/nvfuser-${PIP_WHL_VER_NVFUSER}.dist-info
98+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/nvfuser_common /opt/venv-tritonserver/lib/python3.12/site-packages/nvfuser_common
99+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/nvfuser_direct /opt/venv-tritonserver/lib/python3.12/site-packages/nvfuser_direct
100+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/tools /opt/venv-tritonserver/lib/python3.12/site-packages/tools
101+
# PYCOCOTOOLS
102+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/pycocotools /opt/venv-tritonserver/lib/python3.12/site-packages/pycocotools
103+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/pycocotools-${PIP_WHL_VER_PYCOCOTOOLS}.dist-info /opt/venv-tritonserver/lib/python3.12/site-packages/pycocotools-${PIP_WHL_VER_PYCOCOTOOLS}.dist-info
104+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/ext.cpython-312-*-linux-gnu.so /opt/venv-tritonserver/lib/python3.12/site-packages/ext.cpython-312-*-linux-gnu.so
105+
# PYTORCH_TRITON
106+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/pytorch_triton-${PIP_WHL_VER_PYTORCH_TRITON}.dist-info /opt/venv-tritonserver/lib/python3.12/site-packages/pytorch_triton-${PIP_WHL_VER_PYTORCH_TRITON}.dist-info
107+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/triton /opt/venv-tritonserver/lib/python3.12/site-packages/triton
108+
# TORCH
109+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/functorch /opt/venv-tritonserver/lib/python3.12/site-packages/functorch
110+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torch /opt/venv-tritonserver/lib/python3.12/site-packages/torch
111+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torch-${PIP_WHL_VER_TORCH}.dist-info /opt/venv-tritonserver/lib/python3.12/site-packages/torch-${PIP_WHL_VER_TORCH}.dist-info
112+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchgen /opt/venv-tritonserver/lib/python3.12/site-packages/torchgen
113+
# TORCHAO
114+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchao /opt/venv-tritonserver/lib/python3.12/site-packages/torchao
115+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchao-${PIP_WHL_VER_TORCHAO}.dist-info /opt/venv-tritonserver/lib/python3.12/site-packages/torchao-${PIP_WHL_VER_TORCHAO}.dist-info
116+
# TORCHVISION
117+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchvision /opt/venv-tritonserver/lib/python3.12/site-packages/torchvision
118+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchvision-${PIP_WHL_VER_TORCHVISION}.dist-info /opt/venv-tritonserver/lib/python3.12/site-packages/torchvision-${PIP_WHL_VER_TORCHVISION}.dist-info
119+
COPY --from=pytorch_image /usr/local/lib/python3.12/dist-packages/torchvision.libs /opt/venv-tritonserver/lib/python3.12/site-packages/torchvision.libs
120+
121+
ENV PATH="/opt/venv-tritonserver/bin:${PATH}"
138122
RUN apt-get update -q=2 \
139123
&& apt-get install -y --no-install-recommends \
140-
python3-dev \
141-
python3-pip \
124+
python3.12-dev \
125+
python3.12-minimal \
126+
python3.12-venv \
142127
git-lfs \
143128
perl \
129+
cuda-sanitizer-13-1 \
144130
# Remove previous TRT installation
145131
&& apt-get purge -y "libnvinfer*" \
146-
&& pip3 uninstall -y tensorrt \
132+
&& python3.12 -m venv /opt/venv-tritonserver \
147133
&& rm -rf /var/lib/apt/lists/* \
148-
&& pip3 install --no-cache-dir polygraphy==0.49.9 \
149-
&& curl -sSL https://raw.githubusercontent.com/triton-inference-server/TensorRT-LLM/refs/tags/v1.1.0/docker/common/install_mpi4py.sh | bash
150-
151-
# Install NVRTC
152-
RUN [ "$(uname -m)" != "x86_64" ] && arch="sbsa" || arch="x86_64" \
153-
&& curl -o /tmp/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$arch/cuda-keyring_1.1-1_all.deb \
154-
&& apt install /tmp/cuda-keyring.deb \
155-
&& rm /tmp/cuda-keyring.deb \
156-
&& apt-get remove --purge -y --allow-change-held-packages cuda-nvrtc-dev* \
157-
&& CUDA_VER_SHORT=${CUDA_VER: 0:4} \
158-
&& NVRTC_CUDA_VERSION=${CUDA_VER_SHORT/./-} \
159-
&& apt-get update -qq \
160-
&& apt-get install -y --no-install-recommends \
161-
cuda-nvrtc-dev-${NVRTC_CUDA_VERSION}=${NVRTC_VER} \
162-
libnccl2=${NCCL_VER} \
163-
libnccl-dev=${NCCL_VER} \
164-
libnvshmem3-cuda-13 \
165-
&& dpkg -L libnvshmem3-cuda-13 | grep libnvshmem_host.so | sed -e 's/libnvshmem_host.*//g' | sort -u > /etc/ld.so.conf.d/libnvshmem3-cuda-13.conf \
166-
&& ldconfig \
167-
&& rm -rf /var/lib/apt/lists/*
134+
&& pip3 install --no-cache-dir \
135+
polygraphy==0.49.26 \
136+
networkx \
137+
sympy \
138+
packaging \
139+
opencv-python-headless \
140+
&& curl -sSL https://raw.githubusercontent.com/triton-inference-server/TensorRT-LLM/refs/heads/release/1.2/docker/common/install_mpi4py.sh | bash
141+
142+
RUN find /opt /usr \( -name 'lib*torch*so' -o -name 'libInterceptorInjectionTarget.so' \) -exec dirname {} \; | sort -u > /etc/ld.so.conf.d/tensorrt-llm-pytorch.conf
143+
144+
ENV CUDA_VER=$CUDA_VERSION
168145

146+
FROM pytorch_wrapper AS tensorrt_llm_base
169147
# Install TRT
170148
COPY --from=install_dependencies /usr/local/tensorrt /usr/local/tensorrt
171149
RUN pip3 install /usr/local/tensorrt/python/tensorrt-*-cp$( python3 -c "import sys; print(str(sys.version_info.major) + str(sys.version_info.minor))" )*
@@ -179,7 +157,14 @@ ENV TRT_ROOT=/usr/local/tensorrt
179157
# Install TRT-LLM wheel after all the dependencies are installed
180158
ARG TENSORRTLLM_VER
181159
RUN --mount=type=secret,id=pypi_extra_values,env=PYPI_EXTRA_VALUES \
182-
pip install --no-cache-dir ${PYPI_EXTRA_VALUES} tensorrt_llm==${TENSORRTLLM_VER}
160+
if [ -n "${PYPI_EXTRA_VALUES}" ]; then \
161+
pip3 install --no-cache-dir ${PYPI_EXTRA_VALUES} ; \
162+
else \
163+
pip3 install --no-cache-dir --extra-index-url https://pypi.nvidia.com tensorrt_llm==${TENSORRTLLM_VER} ; \
164+
fi
165+
166+
RUN find /usr /opt -name libtensorrt_llm.so -exec dirname {} \; | sort -u > /etc/ld.so.conf.d/tensorrt-llm.conf \
167+
&& ldconfig
183168

184169
# Copying the Tensorrt LLM scripts and applications
185170
WORKDIR /app

‎tensorrt_llm‎

Submodule tensorrt_llm updated 6640 files

0 commit comments

Comments
 (0)