From b732753be19081f6764bb149910b0f7553c5d48b Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 16 Jun 2026 20:37:12 +0000 Subject: [PATCH 01/11] feat(TRI-1118): add --release-version + PEP 440/817 wheel scheme build_wheel.py (src/python/build_wheel.py): - --release-version CLI flag + TRITON_RELEASE_VERSION env-var fallback. Precedence: CLI flag > env var > in-tree TRITON_VERSION file. CI can pin a release tag without editing source-tree TRITON_VERSION. - Classifies the base version: X.Y.Z (release-semantic) emits PEP 440 normalized output (no local segment, no build tag, no variant label) -- PyPI-clean. Anything else emits a PEP 817 variant label (nv.cu) appended to the wheel filename, keeping the PEP 427 build tag for CI rerun disambiguation. - ANSI-colored info lines surface the chosen path in CI logs. build.py: - Propagates FLAGS.release_version into the wheel build via a single -e TRITON_RELEASE_VERSION= on the docker-run command that drives ./cmake_build inside the tritonserver_buildbase container. For host (--no-container-build) builds, an os.environ.setdefault in main() ensures the cmake_build subprocess inherits the same env. Refs TRI-1118. --- build.py | 12 ++++ src/python/build_wheel.py | 148 +++++++++++++++++++++++++------------- 2 files changed, 112 insertions(+), 48 deletions(-) diff --git a/build.py b/build.py index fa23ee1742..a12b00612f 100755 --- a/build.py +++ b/build.py @@ -1664,6 +1664,11 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ ), ] + # TRI-1118 — propagate the release version into the wheel build + # step. build_wheel.py reads TRITON_RELEASE_VERSION (precedence: + # --release-version flag > env var > TRITON_VERSION file). + runargs += ["-e", f"TRITON_RELEASE_VERSION={FLAGS.release_version}"] + runargs += ["tritonserver_buildbase"] runargs += ["./cmake_build"] @@ -2599,6 +2604,13 @@ def enable_all(): if FLAGS.version is None: FLAGS.version = DEFAULT_TRITON_VERSION_MAP["release_version"] + # TRI-1118 — export the release version so the wheel build picks it up. + # For container builds the value is forwarded into the buildbase via + # `docker run -e` (see create_docker_build_script). For host builds + # (--no-container-build) the cmake_build subprocess inherits this + # process's env, so setting it here covers both paths. + os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.release_version) + if FLAGS.build_parallel is None: FLAGS.build_parallel = multiprocessing.cpu_count() * 2 diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index 27edac6ec5..5e0620a63c 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -34,6 +34,20 @@ import sys from tempfile import mkstemp +# ANSI colors for CI log readability (rendered by GitLab CI, harmlessly +# inert in non-ANSI viewers). Suppressed when stderr isn't a TTY and we +# don't appear to be in CI, or when NO_COLOR is set. +if os.environ.get("NO_COLOR") or not sys.stderr.isatty() and not os.environ.get("CI"): + _GREEN = _YELLOW = _CYAN = _RED = _RESET = "" +else: + _GREEN, _YELLOW, _CYAN, _RED, _RESET = ( + "\033[32m", + "\033[33m", + "\033[36m", + "\033[31m", + "\033[0m", + ) + def fail_if(p, msg): if p: @@ -92,39 +106,32 @@ def _detect_cuda_version() -> str | None: return None -def _compose_version(base_version): - """Compose the full wheel version string. - - Appends a PEP 440 local-version segment describing the NVIDIA - container release and CUDA toolkit so consumers can tell an - nv26.04 wheel from an nv26.05 wheel and a cu132 wheel from a - cu128 wheel. All sources are optional; local non-CI builds return - the version unchanged. - """ +def _compose_variant_label(): + """PEP 817 variant label 'nv.cu'. Returns None + if neither input is detectable or the label violates ^[a-z0-9._]{1,16}$.""" nv = ( os.environ.get("NVIDIA_UPSTREAM_VERSION") or os.environ.get("NVIDIA_TRITON_SERVER_VERSION") or os.environ.get("TRITON_CONTAINER_VERSION") ) cuda = _detect_cuda_version() - print( - f"=== Wheel local-version inputs: " - f"NVIDIA_UPSTREAM_VERSION={os.environ.get('NVIDIA_UPSTREAM_VERSION')!r} " - f"NVIDIA_TRITON_SERVER_VERSION={os.environ.get('NVIDIA_TRITON_SERVER_VERSION')!r} " - f"TRITON_CONTAINER_VERSION={os.environ.get('TRITON_CONTAINER_VERSION')!r} " - f"-> nv={nv!r}, cuda={cuda!r}", - file=sys.stderr, - ) - local = [] + parts = [] if nv: - local.append(f"nv{nv}") + parts.append(f"nv{nv}") if cuda: - parts = cuda.split(".") - if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit(): - local.append(f"cu{parts[0]}{parts[1]}") - if local: - return f"{base_version}+{'.'.join(local)}" - return base_version + cu = cuda.split(".") + if len(cu) >= 2 and cu[0].isdigit() and cu[1].isdigit(): + parts.append(f"cu{cu[0]}{cu[1]}") + if not parts: + return None + label = ".".join(parts) + if len(label) > 16 or not re.fullmatch(r"[a-z0-9._]+", label): + print( + f"{_RED}=== Variant label {label!r} violates PEP 817; skipping{_RESET}", + file=sys.stderr, + ) + return None + return label def _repair_wheel_with_auditwheel(whl_dir, dest_dir): @@ -202,12 +209,40 @@ def main(): required=True, help="Path to Triton Frontend Python binding.", ) + parser.add_argument( + "--release-version", + type=str, + required=False, + default=None, + help=( + "Base PEP 440 release version (e.g. '2.70.0'). Overrides the " + "TRITON_RELEASE_VERSION env var and the in-tree TRITON_VERSION file. " + "Precedence: --release-version > TRITON_RELEASE_VERSION > TRITON_VERSION file." + ), + ) FLAGS = parser.parse_args() - FLAGS.triton_version = None - with open("TRITON_VERSION", "r") as vfile: - FLAGS.triton_version = vfile.readline().strip() + # Base release version source — explicit precedence so CI can pin a + # release tag without editing the in-tree TRITON_VERSION file: + # 1. --release-version CLI flag + # 2. TRITON_RELEASE_VERSION env var + # 3. TRITON_VERSION file in CWD (legacy behaviour) + env_release_version = os.environ.get("TRITON_RELEASE_VERSION") + if FLAGS.release_version: + FLAGS.triton_version = FLAGS.release_version + base_source = "--release-version" + elif env_release_version: + FLAGS.triton_version = env_release_version + base_source = "TRITON_RELEASE_VERSION env" + else: + with open("TRITON_VERSION", "r") as vfile: + FLAGS.triton_version = vfile.readline().strip() + base_source = "TRITON_VERSION file" + print( + f"=== Wheel base version: {FLAGS.triton_version!r} (source: {base_source})", + file=sys.stderr, + ) FLAGS.whl_dir = os.path.join(FLAGS.dest_dir, "wheel") @@ -237,31 +272,29 @@ def main(): os.chdir(FLAGS.whl_dir) print("=== Building wheel") args = ["python3", "setup.py", "bdist_wheel"] - # PEP 427 build tag: lets two wheels of the same version coexist - # (e.g. reruns of the same CI pipeline). Sources, first non-empty - # and usable wins: - # CI_PIPELINE_ID - GitLab pipeline-scoped ID (preferred). - # NVIDIA_BUILD_ID - from build.py's --build-id flag. - # BUILD_NUMBER - generic CI systems. - # PEP 427 requires the build tag to start with a digit. - build_tag = ( - os.environ.get("CI_PIPELINE_ID") - or os.environ.get("NVIDIA_BUILD_ID") - or os.environ.get("BUILD_NUMBER") - ) + + # Release-semantic X.Y.Z -> PyPI-clean (no build tag, no variant label). + # Anything else -> PEP 427 build tag + PEP 817 variant label. + is_release = bool(re.match(r"^\d+\.\d+\.\d+$", FLAGS.triton_version)) print( - f"=== Wheel build-tag inputs: " - f"CI_PIPELINE_ID={os.environ.get('CI_PIPELINE_ID')!r} " - f"NVIDIA_BUILD_ID={os.environ.get('NVIDIA_BUILD_ID')!r} " - f"BUILD_NUMBER={os.environ.get('BUILD_NUMBER')!r} " - f"-> build-tag={build_tag!r}", + f"{_GREEN if is_release else _YELLOW}" + f"=== Version {FLAGS.triton_version!r} -> " + f"{'PEP 440 release (PyPI-clean)' if is_release else 'PEP 817 variant'}" + f"{_RESET}", file=sys.stderr, ) - if build_tag and build_tag != "" and build_tag[:1].isdigit(): - args += [f"--build-number={build_tag}"] + if not is_release: + build_tag = ( + os.environ.get("CI_PIPELINE_ID") + or os.environ.get("NVIDIA_BUILD_ID") + or os.environ.get("BUILD_NUMBER") + ) + if build_tag and build_tag != "" and build_tag[:1].isdigit(): + args += [f"--build-number={build_tag}"] + print(f"{_CYAN}=== PEP 427 build tag: {build_tag}{_RESET}", file=sys.stderr) wenv = os.environ.copy() - wenv["VERSION"] = _compose_version(FLAGS.triton_version) + wenv["VERSION"] = FLAGS.triton_version wenv["TRITON_PYBIND"] = PYBIND_LIB p = subprocess.Popen(args, env=wenv) p.wait() @@ -269,6 +302,25 @@ def main(): _repair_wheel_with_auditwheel(FLAGS.whl_dir, FLAGS.dest_dir) + if not is_release: + label = _compose_variant_label() + if label: + print( + f"{_CYAN}=== PEP 817 variant label: {label!r}{_RESET}", file=sys.stderr + ) + for fname in os.listdir(FLAGS.dest_dir): + if fname.endswith(".whl"): + os.rename( + os.path.join(FLAGS.dest_dir, fname), + os.path.join(FLAGS.dest_dir, fname[:-4] + f"-{label}.whl"), + ) + else: + print( + f"{_RED}=== PEP 817 variant: no nv/cu inputs detected; " + f"wheel emitted unlabeled{_RESET}", + file=sys.stderr, + ) + print(f"=== Output wheel file is in: {FLAGS.dest_dir}") touch(os.path.join(FLAGS.dest_dir, "stamp.whl")) From 32effb12e57e4fcd0c073ea11295f43efb7a112c Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 16 Jun 2026 22:37:17 +0000 Subject: [PATCH 02/11] feat(TRI-1118): gate TRITON_RELEASE_VERSION export on TRITON_VERSION shape Previously `os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.release_version)` fired unconditionally, which forced PEP 440 release-clean wheel naming even when the build was for a dev / pre-release `--version` (e.g. `2.70.0.dev0`) -- the wheel would mismatch the rest of the build. New rule (TRI-1118): --release-version -> always set TRITON_RELEASE_VERSION (explicit override). FLAGS.version is X.Y.Z -> propagate FLAGS.version as TRITON_RELEASE_VERSION (release path). FLAGS.version is anything else -> leave TRITON_RELEASE_VERSION unset so build_wheel.py reads the in-tree TRITON_VERSION file (PEP 817 variant). Also: - `--release-version` default changed from DEFAULT_TRITON_VERSION_MAP ["release_version"] to None so we can detect explicit pass-through. - `docker run -e` now mirrors `os.environ`: only forwarded when set. Refs TRI-1118. --- build.py | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/build.py b/build.py index a12b00612f..59f84eb992 100755 --- a/build.py +++ b/build.py @@ -32,6 +32,7 @@ import os.path import pathlib import platform +import re import stat import subprocess import sys @@ -1664,10 +1665,16 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ ), ] - # TRI-1118 — propagate the release version into the wheel build - # step. build_wheel.py reads TRITON_RELEASE_VERSION (precedence: - # --release-version flag > env var > TRITON_VERSION file). - runargs += ["-e", f"TRITON_RELEASE_VERSION={FLAGS.release_version}"] + # TRI-1118 — propagate TRITON_RELEASE_VERSION into the wheel build + # only when it was actually set in main() (release-semantic version + # or explicit --release-version). Dev / pre-release builds leave it + # unset so build_wheel.py reads the in-tree TRITON_VERSION file and + # takes the PEP 817 variant path. + if "TRITON_RELEASE_VERSION" in os.environ: + runargs += [ + "-e", + f"TRITON_RELEASE_VERSION={os.environ['TRITON_RELEASE_VERSION']}", + ] runargs += ["tritonserver_buildbase"] @@ -2474,8 +2481,12 @@ def enable_all(): parser.add_argument( "--release-version", required=False, - default=DEFAULT_TRITON_VERSION_MAP["release_version"], - help="This flag sets the release version for Triton Inference Server to be built. Default: the latest released version.", + default=None, + help="Override the wheel base version (TRI-1118). When set, exported " + "as TRITON_RELEASE_VERSION so build_wheel.py uses it as the bare " + "PEP 440 version. When unset, build.py falls back to --version when " + "it matches X.Y.Z release-semantic, otherwise leaves the env var " + "unset and lets the in-tree TRITON_VERSION file rule (PEP 817 path).", ) parser.add_argument( "--triton-container-version", @@ -2604,12 +2615,21 @@ def enable_all(): if FLAGS.version is None: FLAGS.version = DEFAULT_TRITON_VERSION_MAP["release_version"] - # TRI-1118 — export the release version so the wheel build picks it up. - # For container builds the value is forwarded into the buildbase via - # `docker run -e` (see create_docker_build_script). For host builds - # (--no-container-build) the cmake_build subprocess inherits this - # process's env, so setting it here covers both paths. - os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.release_version) + # TRI-1118 — choose whether to export TRITON_RELEASE_VERSION based on + # the resolved Triton version: + # - --release-version explicitly passed -> use it (override) + # - FLAGS.version matches X.Y.Z -> propagate FLAGS.version + # - anything else (dev / pre-release / etc) -> leave unset so the + # wheel build reads the + # in-tree TRITON_VERSION + # file (PEP 817 variant). + # The container build forwards this env var via `docker run -e` + # (create_docker_build_script). For --no-container-build, the + # cmake_build subprocess inherits the host env directly. + if FLAGS.release_version is not None: + os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.release_version) + elif re.match(r"^\d+\.\d+\.\d+$", FLAGS.version): + os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.version) if FLAGS.build_parallel is None: FLAGS.build_parallel = multiprocessing.cpu_count() * 2 From dc3cf42dc289ba1c421ae47eb6c8a7dda1cbd641 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 16 Jun 2026 23:09:07 +0000 Subject: [PATCH 03/11] feat(TRI-1118): forward CI_PIPELINE_ID + NVIDIA_UPSTREAM_VERSION to buildbase build_wheel.py reads CI_PIPELINE_ID (for the PEP 427 build tag) and NVIDIA_UPSTREAM_VERSION (for the PEP 817 variant label's nv-part) from the env at wheel-build time, but those vars are set on the GitLab CI runner host -- they were never reaching the tritonserver_buildbase container. Result: dev wheels came out without the build tag and with a fallback nv-part (via NVIDIA_TRITON_SERVER_VERSION ENV in the image). Forward both via `docker run -e`, plus NVIDIA_BUILD_ID as a backup build-tag source for non-CI runs that pass --build-id. CUDA_VERSION is intentionally not forwarded -- the container's own CUDA base image must define it; host CUDA may differ (see comment in core/python/build_wheel.py:_detect_cuda_version). Refs TRI-1118. --- build.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build.py b/build.py index 59f84eb992..d065d4a545 100755 --- a/build.py +++ b/build.py @@ -1675,6 +1675,14 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ "-e", f"TRITON_RELEASE_VERSION={os.environ['TRITON_RELEASE_VERSION']}", ] + # TRI-1118 — forward PEP 427 build-tag and PEP 817 nv-part inputs so + # build_wheel.py inside the buildbase container can emit pipeline-correct + # wheel filenames. CUDA_VERSION is deliberately NOT forwarded -- the + # container's own CUDA base image defines it; host CUDA may differ + # (see core/python/build_wheel.py:_detect_cuda_version). + for var in ("CI_PIPELINE_ID", "NVIDIA_UPSTREAM_VERSION", "NVIDIA_BUILD_ID"): + if os.environ.get(var): + runargs += ["-e", f"{var}={os.environ[var]}"] runargs += ["tritonserver_buildbase"] From 527eb22babfde11b55a26e2faf41588595b81a45 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 16 Jun 2026 23:45:25 +0000 Subject: [PATCH 04/11] feat(TRI-1118): encode CI_PIPELINE_ID via PEP 440 dev counter, drop PEP 427 build tag Mirror of the core change. When the resolved base version matches X.Y.Z.devN and CI_PIPELINE_ID is a positive integer, replace the dev counter with the pipeline id (e.g. 2.70.0.dev0 + CI_PIPELINE_ID=12345 -> 2.70.0.dev12345). The corresponding `--build-number=` injection on `setup.py bdist_wheel` is removed; the pipeline id now lives in the version string itself, where PyPI accepts it and pip can sort by it. Refs TRI-1118. --- src/python/build_wheel.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index 5e0620a63c..49a0060214 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -244,6 +244,21 @@ def main(): file=sys.stderr, ) + # Replace the PEP 440 dev counter with CI_PIPELINE_ID when present, so + # each CI rebuild gets a monotonic, PyPI-uploadable, naturally-sortable + # version (e.g. 2.70.0.dev0 + CI_PIPELINE_ID=12345 -> 2.70.0.dev12345). + # Replaces the legacy PEP 427 build-tag scheme which PyPI rejects. + _pipeline = os.environ.get("CI_PIPELINE_ID", "") + _dev_m = re.match(r"^(\d+\.\d+\.\d+)\.dev\d+$", FLAGS.triton_version) + if _dev_m and _pipeline.isdigit(): + _new = f"{_dev_m.group(1)}.dev{_pipeline}" + print( + f"{_CYAN}=== PEP 440 dev counter: {FLAGS.triton_version!r} -> " + f"{_new!r} (from CI_PIPELINE_ID={_pipeline}){_RESET}", + file=sys.stderr, + ) + FLAGS.triton_version = _new + FLAGS.whl_dir = os.path.join(FLAGS.dest_dir, "wheel") print("=== Building in: {}".format(os.getcwd())) @@ -273,8 +288,10 @@ def main(): print("=== Building wheel") args = ["python3", "setup.py", "bdist_wheel"] - # Release-semantic X.Y.Z -> PyPI-clean (no build tag, no variant label). - # Anything else -> PEP 427 build tag + PEP 817 variant label. + # Release-semantic X.Y.Z -> PyPI-clean (no variant label). + # Anything else -> PEP 817 variant label. The pipeline id is already + # encoded as the PEP 440 .dev counter above, so no separate + # PEP 427 build tag is needed. is_release = bool(re.match(r"^\d+\.\d+\.\d+$", FLAGS.triton_version)) print( f"{_GREEN if is_release else _YELLOW}" @@ -283,15 +300,6 @@ def main(): f"{_RESET}", file=sys.stderr, ) - if not is_release: - build_tag = ( - os.environ.get("CI_PIPELINE_ID") - or os.environ.get("NVIDIA_BUILD_ID") - or os.environ.get("BUILD_NUMBER") - ) - if build_tag and build_tag != "" and build_tag[:1].isdigit(): - args += [f"--build-number={build_tag}"] - print(f"{_CYAN}=== PEP 427 build tag: {build_tag}{_RESET}", file=sys.stderr) wenv = os.environ.copy() wenv["VERSION"] = FLAGS.triton_version From d51e2848f2269d14f796b6a75c6f1749c1d5f0bb Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 17 Jun 2026 00:55:57 +0000 Subject: [PATCH 05/11] fix(TRI-1118): auditwheel writes to dist_dir; relax dev-counter regex Mirror of the core fix. _repair_wheel_with_auditwheel() now writes the manylinux wheel to dist_dir (so CMake's `install(DIRECTORY WHEEL_OUT_DIR)` copies it into /opt/tritonserver/python/) and removes the original linux_ wheel on success. Dev-counter regex relaxed to ^(\d+\.\d+\.\d+)\.?dev\d*$ to accept the legacy `X.Y.Zdev` shape used in the in-tree TRITON_VERSION file. Refs TRI-1118. --- src/python/build_wheel.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index 49a0060214..4732edb964 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -165,8 +165,13 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): for wheel_path in wheels: print(f"=== Running auditwheel repair on {wheel_path}") + # Write the manylinux wheel back into dist_dir so the CMake + # install(DIRECTORY WHEEL_OUT_DIR) line picks it up (it copies + # from /wheel/dist/, not from /). The original + # linux_ wheel is removed on success to keep dist_dir + # canonical — a single manylinux artifact, no duplicates. r = subprocess.run( - ["auditwheel", "repair", wheel_path, "--wheel-dir", dest_dir], + ["auditwheel", "repair", wheel_path, "--wheel-dir", dist_dir], capture_output=True, text=True, ) @@ -177,8 +182,8 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): f"=== Pure-Python wheel detected; falling back to wheel tags " f"({manylinux_tag})" ) - copied = os.path.join(dest_dir, os.path.basename(wheel_path)) - shutil.copy(wheel_path, copied) + # `wheel tags --remove` retags wheel_path in place: writes a + # manylinux-tagged sibling and removes the linux_ input. r2 = subprocess.run( [ "python3", @@ -188,13 +193,17 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): "--platform-tag", manylinux_tag, "--remove", - copied, + wheel_path, ] ) fail_if(r2.returncode != 0, "wheel tags fallback failed") elif r.returncode != 0: sys.stderr.write(r.stderr) fail_if(True, "auditwheel repair failed") + else: + # auditwheel succeeded — drop the pre-repair linux_ + # wheel; only the manylinux wheel should ship. + os.remove(wheel_path) def main(): @@ -248,8 +257,10 @@ def main(): # each CI rebuild gets a monotonic, PyPI-uploadable, naturally-sortable # version (e.g. 2.70.0.dev0 + CI_PIPELINE_ID=12345 -> 2.70.0.dev12345). # Replaces the legacy PEP 427 build-tag scheme which PyPI rejects. + # Regex tolerates both 2.70.0.dev0 (canonical PEP 440) and 2.71.0dev + # (legacy in-tree shape with no period and no counter). _pipeline = os.environ.get("CI_PIPELINE_ID", "") - _dev_m = re.match(r"^(\d+\.\d+\.\d+)\.dev\d+$", FLAGS.triton_version) + _dev_m = re.match(r"^(\d+\.\d+\.\d+)\.?dev\d*$", FLAGS.triton_version) if _dev_m and _pipeline.isdigit(): _new = f"{_dev_m.group(1)}.dev{_pipeline}" print( From 05e228609d1ba14554863409cd474cae35cdee6c Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 17 Jun 2026 01:18:12 +0000 Subject: [PATCH 06/11] fix(TRI-1118): route wheel tagging per PEP 425, omit manylinux when no .so Mirror of the core change. Adds `_wheel_has_so()` and routes: - Has `.so` -> `auditwheel repair` -> PEP 513/599/600 manylinux - No `.so` -> `py3-none-any` (PEP 425 pure-Python); manylinux omitted Replaces the previous hardcoded `manylinux_2_28_` fallback that was emitted whenever auditwheel reported "no ELF" -- which incorrectly attached a manylinux compatibility promise to wheels with no native extensions. Refs TRI-1118. --- src/python/build_wheel.py | 123 +++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index 4732edb964..e7633e3d46 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -32,6 +32,7 @@ import shutil import subprocess import sys +import zipfile from tempfile import mkstemp # ANSI colors for CI log readability (rendered by GitLab CI, harmlessly @@ -134,29 +135,44 @@ def _compose_variant_label(): return label -def _repair_wheel_with_auditwheel(whl_dir, dest_dir): - """Upgrade a linux_ wheel to manylinux_2_X_. - - Ports the pattern established for tritonclient in TRI-286: - 1. auditwheel repair — auto-discovers the minimum manylinux tag - by inspecting glibc symbol requirements of the embedded .so. - 2. python -m wheel tags fallback — used when auditwheel reports - "no ELF" (the wheel has no native extension, e.g. a downstream - build disabled bindings). Mirrors the documented fallback. - 3. No-op with warning — when auditwheel is not installed in the - build image, keep the linux_ wheel as-is so the build - does not regress. +def _wheel_has_so(wheel_path): + """True if the wheel zip contains a native shared library. + + Detects both unversioned (`libfoo.so`) and versioned (`libfoo.so.1.2`) + SONAMEs via filename inspection -- matches what auditwheel and pip + both use to classify wheels. """ - if shutil.which("auditwheel") is None: - print( - "=== WARNING: auditwheel not found on PATH; keeping linux_ " - "wheel as-is. Install auditwheel in the build image to produce " - "PyPI-acceptable manylinux_2_X_ wheels.", - file=sys.stderr, - ) - shutil.copytree(os.path.join(whl_dir, "dist"), dest_dir, dirs_exist_ok=True) - return + with zipfile.ZipFile(wheel_path) as zf: + for name in zf.namelist(): + base = os.path.basename(name) + if base.endswith(".so") or ".so." in base: + return True + return False + +def _repair_wheel_with_auditwheel(whl_dir, dest_dir): + """Apply the correct PEP 425 platform-compatibility tag to each wheel. + + Routing rules (per the relevant PEPs): + - Has native `.so` -> PEP 513 / PEP 599 / PEP 600 `manylinux___` + via `auditwheel repair`. auditwheel inspects the .so's glibc + symbol requirements and picks the lowest manylinux policy that + covers them, then bundles any non-allowlisted dynamic deps. + Original linux_ wheel is removed on success. + - No native `.so` -> PEP 425 pure-Python tag `py3-none-any`. + The manylinux platform tag is OMITTED -- claiming manylinux on + a wheel with no glibc-bound code would be a false compatibility + promise. + + Notes: + - PEP 656 musllinux is not produced here (build containers are + glibc-based; `auditwheel-musl` would be required on musl distros). + - PEP 440 version normalization happens upstream in main(), via + the dev-counter rewrite, before this function runs. + - If a wheel has a `.so` but `auditwheel` is missing from PATH, + the linux_ wheel is kept as-is and a warning is logged + rather than mis-tagging it as manylinux. + """ dist_dir = os.path.join(whl_dir, "dist") wheels = [ os.path.join(dist_dir, w) for w in os.listdir(dist_dir) if w.endswith(".whl") @@ -164,46 +180,55 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): fail_if(not wheels, "no wheel produced by the build") for wheel_path in wheels: - print(f"=== Running auditwheel repair on {wheel_path}") - # Write the manylinux wheel back into dist_dir so the CMake - # install(DIRECTORY WHEEL_OUT_DIR) line picks it up (it copies - # from /wheel/dist/, not from /). The original - # linux_ wheel is removed on success to keep dist_dir - # canonical — a single manylinux artifact, no duplicates. - r = subprocess.run( - ["auditwheel", "repair", wheel_path, "--wheel-dir", dist_dir], - capture_output=True, - text=True, - ) - if r.returncode != 0 and "no ELF" in r.stderr: - arch = os.uname().machine - manylinux_tag = f"manylinux_2_28_{arch}" + if _wheel_has_so(wheel_path): + if shutil.which("auditwheel") is None: + print( + f"{_RED}=== WARNING: native .so found in " + f"{os.path.basename(wheel_path)} but auditwheel not on " + f"PATH; keeping linux_ wheel as-is. Install " + f"auditwheel in the build image to produce " + f"PyPI-acceptable manylinux wheels (PEP 513/599/600).{_RESET}", + file=sys.stderr, + ) + continue print( - f"=== Pure-Python wheel detected; falling back to wheel tags " - f"({manylinux_tag})" + f"{_CYAN}=== Native extension in {os.path.basename(wheel_path)}: " + f"auditwheel repair -> PEP 513/599/600 manylinux{_RESET}", + file=sys.stderr, + ) + r = subprocess.run( + ["auditwheel", "repair", wheel_path, "--wheel-dir", dist_dir], + capture_output=True, + text=True, ) - # `wheel tags --remove` retags wheel_path in place: writes a - # manylinux-tagged sibling and removes the linux_ input. - r2 = subprocess.run( + if r.returncode != 0: + sys.stderr.write(r.stderr) + fail_if(True, "auditwheel repair failed") + os.remove(wheel_path) + else: + print( + f"{_CYAN}=== No native extension in " + f"{os.path.basename(wheel_path)}: retagging as PEP 425 " + f"pure-Python (py3-none-any); manylinux tag omitted{_RESET}", + file=sys.stderr, + ) + r = subprocess.run( [ "python3", "-m", "wheel", "tags", + "--python-tag", + "py3", + "--abi-tag", + "none", "--platform-tag", - manylinux_tag, + "any", "--remove", wheel_path, ] ) - fail_if(r2.returncode != 0, "wheel tags fallback failed") - elif r.returncode != 0: - sys.stderr.write(r.stderr) - fail_if(True, "auditwheel repair failed") - else: - # auditwheel succeeded — drop the pre-repair linux_ - # wheel; only the manylinux wheel should ship. - os.remove(wheel_path) + fail_if(r.returncode != 0, "wheel tags retag failed for pure-Python wheel") def main(): From a47acd786e64bd569ad75fafe39845ddc40e4544 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 17 Jun 2026 01:50:52 +0000 Subject: [PATCH 07/11] feat(TRI-1118): install auditwheel in both buildbase Dockerfiles build_wheel.py needs auditwheel to apply PEP 513/599/600 manylinux tags to wheels containing native .so files. Previously the tool's presence relied entirely on the BASE_IMAGE: - py3-base-rhel (cuda:*-manylinux-* base) -- shipped with auditwheel by manylinux convention. Worked. - py3-base (cuda:*-ubuntu* base) -- did NOT ship auditwheel. Wheels with .so silently stayed at the raw linux_ tag and were rejected by PyPI / unusable from Artifactory PyPI mirrors. Add `auditwheel` to the pip install list in both create_dockerfile_buildbase and create_dockerfile_buildbase_rhel so every buildbase image has it regardless of base. Unpinned -- recent auditwheel releases all understand PEP 600 manylinux_X_Y semantics; pin if a regression appears. Refs TRI-1118. --- build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.py b/build.py index d065d4a545..fad526e7b7 100755 --- a/build.py +++ b/build.py @@ -922,6 +922,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): RUN pip3 install --upgrade pip \\ && pip3 install --upgrade \\ + auditwheel \\ build \\ wheel \\ setuptools \\ @@ -1035,6 +1036,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade \\ + auditwheel \\ build \\ docker \\ virtualenv \\ From 6eafccf27d49cdb6e859d1b4edea16a9b3a2d183 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:49:02 +0000 Subject: [PATCH 08/11] fix(TRI-1118): prevent compressed manylinux tag set from double-repair Mirror of the core fix. Symptom seen in CI: tritonserver-2.70.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl Cause: CMake invokes the wheel custom command twice (build + install phases); the second pass finds the manylinux wheel left over from the first pass alongside the newly-built linux_ wheel, and runs auditwheel on both. auditwheel applied to an already-tagged wheel emits a compressed PEP 425 tag set. Fixes: 1. Clean wheel/dist/ before `setup.py bdist_wheel` so leftovers from a previous CMake pass can't accumulate. 2. In _repair_wheel_with_auditwheel, skip wheels whose filename already carries a manylinux or musllinux tag. Refs TRI-1118. --- src/python/build_wheel.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index e7633e3d46..63286688da 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -180,6 +180,19 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): fail_if(not wheels, "no wheel produced by the build") for wheel_path in wheels: + fname = os.path.basename(wheel_path) + # Skip wheels that already carry a manylinux/musllinux platform + # tag. Re-running auditwheel on an already-repaired wheel produces + # a compressed PEP 425 tag set + # (e.g. manylinux_2_27_x86_64.manylinux_2_28_x86_64) -- valid but + # noisy. This guards against CMake invoking this custom command + # twice (build + install phases) and finding stale wheels in dist/. + if "manylinux" in fname or "musllinux" in fname: + print( + f"{_CYAN}=== Skipping already-tagged wheel: {fname}{_RESET}", + file=sys.stderr, + ) + continue if _wheel_has_so(wheel_path): if shutil.which("auditwheel") is None: print( @@ -321,6 +334,15 @@ def main(): shutil.copyfile("setup.py", os.path.join(FLAGS.whl_dir, "setup.py")) os.chdir(FLAGS.whl_dir) + # Clean dist/ to prevent accumulating wheels from prior runs. CMake may + # invoke this custom command twice (build + install phases); without + # this, dist/ would end up with the linux_ wheel just produced + # AND the manylinux___ wheel left over from the previous + # run, and _repair_wheel_with_auditwheel would process both, producing + # wheels with compressed PEP 425 tag sets. + _dist = os.path.join(FLAGS.whl_dir, "dist") + if os.path.isdir(_dist): + shutil.rmtree(_dist) print("=== Building wheel") args = ["python3", "setup.py", "bdist_wheel"] From a973665a99d459f21a53cd2506ea426da00035e3 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:47:44 +0000 Subject: [PATCH 09/11] fix(TRI-1118): normalize compressed manylinux tag sets to highest version Mirror of the core change. If a wheel ends up with a compressed PEP 425 platform-tag set containing multiple manylinux entries, collapse to the highest manylinux version (strictest glibc baseline). Refs TRI-1118. --- src/python/build_wheel.py | 65 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index 63286688da..9ddc21dc5f 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -135,6 +135,67 @@ def _compose_variant_label(): return label +def _normalize_to_highest_manylinux(dist_dir): + """Collapse compressed manylinux tag sets to the highest version. + + auditwheel may emit a wheel whose PEP 425 platform-tag component + contains multiple manylinux entries joined by `.`, e.g. + `manylinux_2_27_x86_64.manylinux_2_28_x86_64`. Per project policy + (TRI-1118), keep only the highest version -- the strictest glibc + baseline. + + No-op for wheels already carrying a single platform tag. Other + non-manylinux entries in the compressed set are preserved. + """ + manylinux_re = re.compile(r"^manylinux_(\d+)_(\d+)_(.+)$") + for fname in os.listdir(dist_dir): + if not fname.endswith(".whl"): + continue + parts = fname[:-4].split("-") + if len(parts) < 5: + continue + plat = parts[-1] + if "." not in plat: + continue + tags = plat.split(".") + manylinux_tags = [] + other_tags = [] + for t in tags: + m = manylinux_re.match(t) + if m: + manylinux_tags.append(((int(m.group(1)), int(m.group(2))), t)) + else: + other_tags.append(t) + if len(manylinux_tags) <= 1 and not other_tags: + continue + if not manylinux_tags: + continue + manylinux_tags.sort() + highest = manylinux_tags[-1][1] + new_plat = ".".join([highest] + other_tags) if other_tags else highest + if new_plat == plat: + continue + wheel_path = os.path.join(dist_dir, fname) + print( + f"{_CYAN}=== Compressed platform tag in {fname!r}: " + f"{plat!r} -> {new_plat!r} (highest manylinux){_RESET}", + file=sys.stderr, + ) + r = subprocess.run( + [ + "python3", + "-m", + "wheel", + "tags", + "--platform-tag", + new_plat, + "--remove", + wheel_path, + ] + ) + fail_if(r.returncode != 0, "wheel tags normalization failed") + + def _wheel_has_so(wheel_path): """True if the wheel zip contains a native shared library. @@ -243,6 +304,10 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): ) fail_if(r.returncode != 0, "wheel tags retag failed for pure-Python wheel") + # Post-process: if any resulting wheel carries a compressed manylinux + # tag set, collapse it to the highest version (project policy). + _normalize_to_highest_manylinux(dist_dir) + def main(): parser = argparse.ArgumentParser() From 13675ec107011b4edddeea43cc3d7345f034567d Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:46:20 +0000 Subject: [PATCH 10/11] fix(TRI-1118): pull tritonfrontend wheel into install dir before cp glob Symptom (pipeline 55073480 / py3-wheel-publish): the publish job's `jf rt upload 'tritonfrontend-*.whl'` finds no matches in /opt/tritonserver/python/ inside the base image and aborts with `[Error] No errors, but also no files affected (fail-no-op flag)`. Cause: in core_build(), the existing `cp .../install/python/triton*.whl .../install/python/` line is supposed to pick up BOTH the core's tritonserver wheel AND the server's tritonfrontend wheel. The glob is expanded at shell-execution time against repo_install_dir/python/. The trace shows only `tritonserver-*-manylinux_2_39_x86_64.whl` ever lands there -- CMake's `install(DIRECTORY ${WHEEL_OUT_DIR})` rule for tritonfrontend (server/src/python/CMakeLists.txt) lives in the triton-server sub-build (build/triton-server/python/) and is not traversed by the outer install pass that handles the core wheel from _deps/repo-core-build. Result: tritonfrontend wheel is built but never copied to the base image's /opt/tritonserver/python/, and the publish job has nothing to upload. Workaround in build.py rather than CMake: after `makeinstall()`, run `find -path '*/wheel/dist/tritonfrontend-*.whl' -exec cp {} /python/ \;` so the wheel lands in the same directory the subsequent `triton*.whl` glob already targets. Safe no-op when the wheel is absent (downstream builds with frontend disabled). Defensive against both build locations the wheel could end up in across CMake layouts. Refs TRI-1118. --- build.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/build.py b/build.py index fad526e7b7..8e527833ad 100755 --- a/build.py +++ b/build.py @@ -1809,6 +1809,18 @@ def core_build( # [FIXME] Placing the tritonserver and tritonfrontend wheel files in 'python' for now, # should be uploaded to pip registry to be able to install directly cmake_script.mkdir(os.path.join(install_dir, "python")) + # TRI-1118 — tritonfrontend wheel is built by the triton-server + # sub-build's `frontend-server-wheel` target, but its CMake + # `install(DIRECTORY ${WHEEL_OUT_DIR})` rule doesn't traverse the + # outer install pass, so the wheel never lands in + # repo_install_dir/python/ alongside the core's tritonserver wheel. + # Pull it in from the build tree directly so the subsequent + # `triton*.whl` glob picks it up. Safe no-op when the wheel is + # absent (e.g. downstream builds that disable the frontend). + cmake_script.cmd( + f"find {repo_build_dir} -path '*/wheel/dist/tritonfrontend-*.whl' " + f"-exec cp {{}} {os.path.join(repo_install_dir, 'python')}/ \\;" + ) cmake_script.cp( os.path.join(repo_install_dir, "python", "triton*.whl"), os.path.join(install_dir, "python"), From decafa680ca111ee3fbc9ecaeb12835a28270f60 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:49:16 +0000 Subject: [PATCH 11/11] fix(TRI-1118): fail-fast on missing wheels in base-image pip install The legacy `find /opt/tritonserver/python ... | xargs -I {} pip install` silently no-ops when find produces zero matches: xargs runs the command zero times and the layer succeeds. That masked the TRI-1118 install-dir gap (pipeline 55073480) -- tritonfrontend wheel never reached /opt/tritonserver/python, the layer succeeded, and the publish job was the first to notice ("no files affected"). Replace with an explicit existence check per package, then xargs the list through pip. A missing wheel now breaks the Docker build at this layer with a clear error, rather than as a silent downstream regression. Defensive layer on top of the cp-time fix in core_build() (commit 13675ec10). Refs TRI-1118. --- build.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/build.py b/build.py index 8e527833ad..e8b76c3080 100755 --- a/build.py +++ b/build.py @@ -1174,10 +1174,21 @@ def create_dockerfile_linux( WORKDIR /opt/tritonserver COPY NVIDIA_Deep_Learning_Container_License.pdf . -RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonserver-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] && \\ - find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonfrontend-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] +# TRI-1118 — fail fast if either tritonserver or tritonfrontend wheel is +# missing from /opt/tritonserver/python. The legacy `find | xargs -I` is +# a silent no-op when find returns zero matches: xargs runs the command +# zero times and the layer succeeds, masking the gap until something +# downstream (a wheel publish job, or pip install tritonfrontend) +# discovers nothing was actually installed. Check existence first. +RUN set -e; \\ + for pkg in tritonserver tritonfrontend; do \\ + wheels=$(find /opt/tritonserver/python -maxdepth 1 -type f -name "${{pkg}}-*.whl"); \\ + if [ -z "$wheels" ]; then \\ + echo "ERROR: ${{pkg}}-*.whl missing from /opt/tritonserver/python -- build did not stage the wheel into the image" >&2; \\ + exit 1; \\ + fi; \\ + printf '%s\\n' "$wheels" | xargs -I {{}} pip install --upgrade "{{}}[{FLAGS.triton_wheels_dependencies_group}]"; \\ + done RUN pip3 install -r python/openai/requirements.txt