diff --git a/build.py b/build.py index fa23ee1742..e8b76c3080 100755 --- a/build.py +++ b/build.py @@ -32,6 +32,7 @@ import os.path import pathlib import platform +import re import stat import subprocess import sys @@ -921,6 +922,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): RUN pip3 install --upgrade pip \\ && pip3 install --upgrade \\ + auditwheel \\ build \\ wheel \\ setuptools \\ @@ -1034,6 +1036,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade \\ + auditwheel \\ build \\ docker \\ virtualenv \\ @@ -1171,10 +1174,21 @@ def create_dockerfile_linux( WORKDIR /opt/tritonserver COPY NVIDIA_Deep_Learning_Container_License.pdf . -RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonserver-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] && \\ - find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonfrontend-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] +# TRI-1118 — fail fast if either tritonserver or tritonfrontend wheel is +# missing from /opt/tritonserver/python. The legacy `find | xargs -I` is +# a silent no-op when find returns zero matches: xargs runs the command +# zero times and the layer succeeds, masking the gap until something +# downstream (a wheel publish job, or pip install tritonfrontend) +# discovers nothing was actually installed. Check existence first. +RUN set -e; \\ + for pkg in tritonserver tritonfrontend; do \\ + wheels=$(find /opt/tritonserver/python -maxdepth 1 -type f -name "${{pkg}}-*.whl"); \\ + if [ -z "$wheels" ]; then \\ + echo "ERROR: ${{pkg}}-*.whl missing from /opt/tritonserver/python -- build did not stage the wheel into the image" >&2; \\ + exit 1; \\ + fi; \\ + printf '%s\\n' "$wheels" | xargs -I {{}} pip install --upgrade "{{}}[{FLAGS.triton_wheels_dependencies_group}]"; \\ + done RUN pip3 install -r python/openai/requirements.txt @@ -1664,6 +1678,25 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ ), ] + # TRI-1118 — propagate TRITON_RELEASE_VERSION into the wheel build + # only when it was actually set in main() (release-semantic version + # or explicit --release-version). Dev / pre-release builds leave it + # unset so build_wheel.py reads the in-tree TRITON_VERSION file and + # takes the PEP 817 variant path. + if "TRITON_RELEASE_VERSION" in os.environ: + runargs += [ + "-e", + f"TRITON_RELEASE_VERSION={os.environ['TRITON_RELEASE_VERSION']}", + ] + # TRI-1118 — forward PEP 427 build-tag and PEP 817 nv-part inputs so + # build_wheel.py inside the buildbase container can emit pipeline-correct + # wheel filenames. CUDA_VERSION is deliberately NOT forwarded -- the + # container's own CUDA base image defines it; host CUDA may differ + # (see core/python/build_wheel.py:_detect_cuda_version). + for var in ("CI_PIPELINE_ID", "NVIDIA_UPSTREAM_VERSION", "NVIDIA_BUILD_ID"): + if os.environ.get(var): + runargs += ["-e", f"{var}={os.environ[var]}"] + runargs += ["tritonserver_buildbase"] runargs += ["./cmake_build"] @@ -1787,6 +1820,18 @@ def core_build( # [FIXME] Placing the tritonserver and tritonfrontend wheel files in 'python' for now, # should be uploaded to pip registry to be able to install directly cmake_script.mkdir(os.path.join(install_dir, "python")) + # TRI-1118 — tritonfrontend wheel is built by the triton-server + # sub-build's `frontend-server-wheel` target, but its CMake + # `install(DIRECTORY ${WHEEL_OUT_DIR})` rule doesn't traverse the + # outer install pass, so the wheel never lands in + # repo_install_dir/python/ alongside the core's tritonserver wheel. + # Pull it in from the build tree directly so the subsequent + # `triton*.whl` glob picks it up. Safe no-op when the wheel is + # absent (e.g. downstream builds that disable the frontend). + cmake_script.cmd( + f"find {repo_build_dir} -path '*/wheel/dist/tritonfrontend-*.whl' " + f"-exec cp {{}} {os.path.join(repo_install_dir, 'python')}/ \\;" + ) cmake_script.cp( os.path.join(repo_install_dir, "python", "triton*.whl"), os.path.join(install_dir, "python"), @@ -2469,8 +2514,12 @@ def enable_all(): parser.add_argument( "--release-version", required=False, - default=DEFAULT_TRITON_VERSION_MAP["release_version"], - help="This flag sets the release version for Triton Inference Server to be built. Default: the latest released version.", + default=None, + help="Override the wheel base version (TRI-1118). When set, exported " + "as TRITON_RELEASE_VERSION so build_wheel.py uses it as the bare " + "PEP 440 version. When unset, build.py falls back to --version when " + "it matches X.Y.Z release-semantic, otherwise leaves the env var " + "unset and lets the in-tree TRITON_VERSION file rule (PEP 817 path).", ) parser.add_argument( "--triton-container-version", @@ -2599,6 +2648,22 @@ def enable_all(): if FLAGS.version is None: FLAGS.version = DEFAULT_TRITON_VERSION_MAP["release_version"] + # TRI-1118 — choose whether to export TRITON_RELEASE_VERSION based on + # the resolved Triton version: + # - --release-version explicitly passed -> use it (override) + # - FLAGS.version matches X.Y.Z -> propagate FLAGS.version + # - anything else (dev / pre-release / etc) -> leave unset so the + # wheel build reads the + # in-tree TRITON_VERSION + # file (PEP 817 variant). + # The container build forwards this env var via `docker run -e` + # (create_docker_build_script). For --no-container-build, the + # cmake_build subprocess inherits the host env directly. + if FLAGS.release_version is not None: + os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.release_version) + elif re.match(r"^\d+\.\d+\.\d+$", FLAGS.version): + os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.version) + if FLAGS.build_parallel is None: FLAGS.build_parallel = multiprocessing.cpu_count() * 2 diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index 27edac6ec5..9ddc21dc5f 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -32,8 +32,23 @@ import shutil import subprocess import sys +import zipfile from tempfile import mkstemp +# ANSI colors for CI log readability (rendered by GitLab CI, harmlessly +# inert in non-ANSI viewers). Suppressed when stderr isn't a TTY and we +# don't appear to be in CI, or when NO_COLOR is set. +if os.environ.get("NO_COLOR") or not sys.stderr.isatty() and not os.environ.get("CI"): + _GREEN = _YELLOW = _CYAN = _RED = _RESET = "" +else: + _GREEN, _YELLOW, _CYAN, _RED, _RESET = ( + "\033[32m", + "\033[33m", + "\033[36m", + "\033[31m", + "\033[0m", + ) + def fail_if(p, msg): if p: @@ -92,64 +107,133 @@ def _detect_cuda_version() -> str | None: return None -def _compose_version(base_version): - """Compose the full wheel version string. - - Appends a PEP 440 local-version segment describing the NVIDIA - container release and CUDA toolkit so consumers can tell an - nv26.04 wheel from an nv26.05 wheel and a cu132 wheel from a - cu128 wheel. All sources are optional; local non-CI builds return - the version unchanged. - """ +def _compose_variant_label(): + """PEP 817 variant label 'nv.cu'. Returns None + if neither input is detectable or the label violates ^[a-z0-9._]{1,16}$.""" nv = ( os.environ.get("NVIDIA_UPSTREAM_VERSION") or os.environ.get("NVIDIA_TRITON_SERVER_VERSION") or os.environ.get("TRITON_CONTAINER_VERSION") ) cuda = _detect_cuda_version() - print( - f"=== Wheel local-version inputs: " - f"NVIDIA_UPSTREAM_VERSION={os.environ.get('NVIDIA_UPSTREAM_VERSION')!r} " - f"NVIDIA_TRITON_SERVER_VERSION={os.environ.get('NVIDIA_TRITON_SERVER_VERSION')!r} " - f"TRITON_CONTAINER_VERSION={os.environ.get('TRITON_CONTAINER_VERSION')!r} " - f"-> nv={nv!r}, cuda={cuda!r}", - file=sys.stderr, - ) - local = [] + parts = [] if nv: - local.append(f"nv{nv}") + parts.append(f"nv{nv}") if cuda: - parts = cuda.split(".") - if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit(): - local.append(f"cu{parts[0]}{parts[1]}") - if local: - return f"{base_version}+{'.'.join(local)}" - return base_version + cu = cuda.split(".") + if len(cu) >= 2 and cu[0].isdigit() and cu[1].isdigit(): + parts.append(f"cu{cu[0]}{cu[1]}") + if not parts: + return None + label = ".".join(parts) + if len(label) > 16 or not re.fullmatch(r"[a-z0-9._]+", label): + print( + f"{_RED}=== Variant label {label!r} violates PEP 817; skipping{_RESET}", + file=sys.stderr, + ) + return None + return label -def _repair_wheel_with_auditwheel(whl_dir, dest_dir): - """Upgrade a linux_ wheel to manylinux_2_X_. - - Ports the pattern established for tritonclient in TRI-286: - 1. auditwheel repair — auto-discovers the minimum manylinux tag - by inspecting glibc symbol requirements of the embedded .so. - 2. python -m wheel tags fallback — used when auditwheel reports - "no ELF" (the wheel has no native extension, e.g. a downstream - build disabled bindings). Mirrors the documented fallback. - 3. No-op with warning — when auditwheel is not installed in the - build image, keep the linux_ wheel as-is so the build - does not regress. +def _normalize_to_highest_manylinux(dist_dir): + """Collapse compressed manylinux tag sets to the highest version. + + auditwheel may emit a wheel whose PEP 425 platform-tag component + contains multiple manylinux entries joined by `.`, e.g. + `manylinux_2_27_x86_64.manylinux_2_28_x86_64`. Per project policy + (TRI-1118), keep only the highest version -- the strictest glibc + baseline. + + No-op for wheels already carrying a single platform tag. Other + non-manylinux entries in the compressed set are preserved. """ - if shutil.which("auditwheel") is None: + manylinux_re = re.compile(r"^manylinux_(\d+)_(\d+)_(.+)$") + for fname in os.listdir(dist_dir): + if not fname.endswith(".whl"): + continue + parts = fname[:-4].split("-") + if len(parts) < 5: + continue + plat = parts[-1] + if "." not in plat: + continue + tags = plat.split(".") + manylinux_tags = [] + other_tags = [] + for t in tags: + m = manylinux_re.match(t) + if m: + manylinux_tags.append(((int(m.group(1)), int(m.group(2))), t)) + else: + other_tags.append(t) + if len(manylinux_tags) <= 1 and not other_tags: + continue + if not manylinux_tags: + continue + manylinux_tags.sort() + highest = manylinux_tags[-1][1] + new_plat = ".".join([highest] + other_tags) if other_tags else highest + if new_plat == plat: + continue + wheel_path = os.path.join(dist_dir, fname) print( - "=== WARNING: auditwheel not found on PATH; keeping linux_ " - "wheel as-is. Install auditwheel in the build image to produce " - "PyPI-acceptable manylinux_2_X_ wheels.", + f"{_CYAN}=== Compressed platform tag in {fname!r}: " + f"{plat!r} -> {new_plat!r} (highest manylinux){_RESET}", file=sys.stderr, ) - shutil.copytree(os.path.join(whl_dir, "dist"), dest_dir, dirs_exist_ok=True) - return + r = subprocess.run( + [ + "python3", + "-m", + "wheel", + "tags", + "--platform-tag", + new_plat, + "--remove", + wheel_path, + ] + ) + fail_if(r.returncode != 0, "wheel tags normalization failed") + + +def _wheel_has_so(wheel_path): + """True if the wheel zip contains a native shared library. + Detects both unversioned (`libfoo.so`) and versioned (`libfoo.so.1.2`) + SONAMEs via filename inspection -- matches what auditwheel and pip + both use to classify wheels. + """ + with zipfile.ZipFile(wheel_path) as zf: + for name in zf.namelist(): + base = os.path.basename(name) + if base.endswith(".so") or ".so." in base: + return True + return False + + +def _repair_wheel_with_auditwheel(whl_dir, dest_dir): + """Apply the correct PEP 425 platform-compatibility tag to each wheel. + + Routing rules (per the relevant PEPs): + - Has native `.so` -> PEP 513 / PEP 599 / PEP 600 `manylinux___` + via `auditwheel repair`. auditwheel inspects the .so's glibc + symbol requirements and picks the lowest manylinux policy that + covers them, then bundles any non-allowlisted dynamic deps. + Original linux_ wheel is removed on success. + - No native `.so` -> PEP 425 pure-Python tag `py3-none-any`. + The manylinux platform tag is OMITTED -- claiming manylinux on + a wheel with no glibc-bound code would be a false compatibility + promise. + + Notes: + - PEP 656 musllinux is not produced here (build containers are + glibc-based; `auditwheel-musl` would be required on musl distros). + - PEP 440 version normalization happens upstream in main(), via + the dev-counter rewrite, before this function runs. + - If a wheel has a `.so` but `auditwheel` is missing from PATH, + the linux_ wheel is kept as-is and a warning is logged + rather than mis-tagging it as manylinux. + """ dist_dir = os.path.join(whl_dir, "dist") wheels = [ os.path.join(dist_dir, w) for w in os.listdir(dist_dir) if w.endswith(".whl") @@ -157,37 +241,72 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): fail_if(not wheels, "no wheel produced by the build") for wheel_path in wheels: - print(f"=== Running auditwheel repair on {wheel_path}") - r = subprocess.run( - ["auditwheel", "repair", wheel_path, "--wheel-dir", dest_dir], - capture_output=True, - text=True, - ) - if r.returncode != 0 and "no ELF" in r.stderr: - arch = os.uname().machine - manylinux_tag = f"manylinux_2_28_{arch}" + fname = os.path.basename(wheel_path) + # Skip wheels that already carry a manylinux/musllinux platform + # tag. Re-running auditwheel on an already-repaired wheel produces + # a compressed PEP 425 tag set + # (e.g. manylinux_2_27_x86_64.manylinux_2_28_x86_64) -- valid but + # noisy. This guards against CMake invoking this custom command + # twice (build + install phases) and finding stale wheels in dist/. + if "manylinux" in fname or "musllinux" in fname: + print( + f"{_CYAN}=== Skipping already-tagged wheel: {fname}{_RESET}", + file=sys.stderr, + ) + continue + if _wheel_has_so(wheel_path): + if shutil.which("auditwheel") is None: + print( + f"{_RED}=== WARNING: native .so found in " + f"{os.path.basename(wheel_path)} but auditwheel not on " + f"PATH; keeping linux_ wheel as-is. Install " + f"auditwheel in the build image to produce " + f"PyPI-acceptable manylinux wheels (PEP 513/599/600).{_RESET}", + file=sys.stderr, + ) + continue + print( + f"{_CYAN}=== Native extension in {os.path.basename(wheel_path)}: " + f"auditwheel repair -> PEP 513/599/600 manylinux{_RESET}", + file=sys.stderr, + ) + r = subprocess.run( + ["auditwheel", "repair", wheel_path, "--wheel-dir", dist_dir], + capture_output=True, + text=True, + ) + if r.returncode != 0: + sys.stderr.write(r.stderr) + fail_if(True, "auditwheel repair failed") + os.remove(wheel_path) + else: print( - f"=== Pure-Python wheel detected; falling back to wheel tags " - f"({manylinux_tag})" + f"{_CYAN}=== No native extension in " + f"{os.path.basename(wheel_path)}: retagging as PEP 425 " + f"pure-Python (py3-none-any); manylinux tag omitted{_RESET}", + file=sys.stderr, ) - copied = os.path.join(dest_dir, os.path.basename(wheel_path)) - shutil.copy(wheel_path, copied) - r2 = subprocess.run( + r = subprocess.run( [ "python3", "-m", "wheel", "tags", + "--python-tag", + "py3", + "--abi-tag", + "none", "--platform-tag", - manylinux_tag, + "any", "--remove", - copied, + wheel_path, ] ) - fail_if(r2.returncode != 0, "wheel tags fallback failed") - elif r.returncode != 0: - sys.stderr.write(r.stderr) - fail_if(True, "auditwheel repair failed") + fail_if(r.returncode != 0, "wheel tags retag failed for pure-Python wheel") + + # Post-process: if any resulting wheel carries a compressed manylinux + # tag set, collapse it to the highest version (project policy). + _normalize_to_highest_manylinux(dist_dir) def main(): @@ -202,12 +321,57 @@ def main(): required=True, help="Path to Triton Frontend Python binding.", ) + parser.add_argument( + "--release-version", + type=str, + required=False, + default=None, + help=( + "Base PEP 440 release version (e.g. '2.70.0'). Overrides the " + "TRITON_RELEASE_VERSION env var and the in-tree TRITON_VERSION file. " + "Precedence: --release-version > TRITON_RELEASE_VERSION > TRITON_VERSION file." + ), + ) FLAGS = parser.parse_args() - FLAGS.triton_version = None - with open("TRITON_VERSION", "r") as vfile: - FLAGS.triton_version = vfile.readline().strip() + # Base release version source — explicit precedence so CI can pin a + # release tag without editing the in-tree TRITON_VERSION file: + # 1. --release-version CLI flag + # 2. TRITON_RELEASE_VERSION env var + # 3. TRITON_VERSION file in CWD (legacy behaviour) + env_release_version = os.environ.get("TRITON_RELEASE_VERSION") + if FLAGS.release_version: + FLAGS.triton_version = FLAGS.release_version + base_source = "--release-version" + elif env_release_version: + FLAGS.triton_version = env_release_version + base_source = "TRITON_RELEASE_VERSION env" + else: + with open("TRITON_VERSION", "r") as vfile: + FLAGS.triton_version = vfile.readline().strip() + base_source = "TRITON_VERSION file" + print( + f"=== Wheel base version: {FLAGS.triton_version!r} (source: {base_source})", + file=sys.stderr, + ) + + # Replace the PEP 440 dev counter with CI_PIPELINE_ID when present, so + # each CI rebuild gets a monotonic, PyPI-uploadable, naturally-sortable + # version (e.g. 2.70.0.dev0 + CI_PIPELINE_ID=12345 -> 2.70.0.dev12345). + # Replaces the legacy PEP 427 build-tag scheme which PyPI rejects. + # Regex tolerates both 2.70.0.dev0 (canonical PEP 440) and 2.71.0dev + # (legacy in-tree shape with no period and no counter). + _pipeline = os.environ.get("CI_PIPELINE_ID", "") + _dev_m = re.match(r"^(\d+\.\d+\.\d+)\.?dev\d*$", FLAGS.triton_version) + if _dev_m and _pipeline.isdigit(): + _new = f"{_dev_m.group(1)}.dev{_pipeline}" + print( + f"{_CYAN}=== PEP 440 dev counter: {FLAGS.triton_version!r} -> " + f"{_new!r} (from CI_PIPELINE_ID={_pipeline}){_RESET}", + file=sys.stderr, + ) + FLAGS.triton_version = _new FLAGS.whl_dir = os.path.join(FLAGS.dest_dir, "wheel") @@ -235,33 +399,33 @@ def main(): shutil.copyfile("setup.py", os.path.join(FLAGS.whl_dir, "setup.py")) os.chdir(FLAGS.whl_dir) + # Clean dist/ to prevent accumulating wheels from prior runs. CMake may + # invoke this custom command twice (build + install phases); without + # this, dist/ would end up with the linux_ wheel just produced + # AND the manylinux___ wheel left over from the previous + # run, and _repair_wheel_with_auditwheel would process both, producing + # wheels with compressed PEP 425 tag sets. + _dist = os.path.join(FLAGS.whl_dir, "dist") + if os.path.isdir(_dist): + shutil.rmtree(_dist) print("=== Building wheel") args = ["python3", "setup.py", "bdist_wheel"] - # PEP 427 build tag: lets two wheels of the same version coexist - # (e.g. reruns of the same CI pipeline). Sources, first non-empty - # and usable wins: - # CI_PIPELINE_ID - GitLab pipeline-scoped ID (preferred). - # NVIDIA_BUILD_ID - from build.py's --build-id flag. - # BUILD_NUMBER - generic CI systems. - # PEP 427 requires the build tag to start with a digit. - build_tag = ( - os.environ.get("CI_PIPELINE_ID") - or os.environ.get("NVIDIA_BUILD_ID") - or os.environ.get("BUILD_NUMBER") - ) + + # Release-semantic X.Y.Z -> PyPI-clean (no variant label). + # Anything else -> PEP 817 variant label. The pipeline id is already + # encoded as the PEP 440 .dev counter above, so no separate + # PEP 427 build tag is needed. + is_release = bool(re.match(r"^\d+\.\d+\.\d+$", FLAGS.triton_version)) print( - f"=== Wheel build-tag inputs: " - f"CI_PIPELINE_ID={os.environ.get('CI_PIPELINE_ID')!r} " - f"NVIDIA_BUILD_ID={os.environ.get('NVIDIA_BUILD_ID')!r} " - f"BUILD_NUMBER={os.environ.get('BUILD_NUMBER')!r} " - f"-> build-tag={build_tag!r}", + f"{_GREEN if is_release else _YELLOW}" + f"=== Version {FLAGS.triton_version!r} -> " + f"{'PEP 440 release (PyPI-clean)' if is_release else 'PEP 817 variant'}" + f"{_RESET}", file=sys.stderr, ) - if build_tag and build_tag != "" and build_tag[:1].isdigit(): - args += [f"--build-number={build_tag}"] wenv = os.environ.copy() - wenv["VERSION"] = _compose_version(FLAGS.triton_version) + wenv["VERSION"] = FLAGS.triton_version wenv["TRITON_PYBIND"] = PYBIND_LIB p = subprocess.Popen(args, env=wenv) p.wait() @@ -269,6 +433,25 @@ def main(): _repair_wheel_with_auditwheel(FLAGS.whl_dir, FLAGS.dest_dir) + if not is_release: + label = _compose_variant_label() + if label: + print( + f"{_CYAN}=== PEP 817 variant label: {label!r}{_RESET}", file=sys.stderr + ) + for fname in os.listdir(FLAGS.dest_dir): + if fname.endswith(".whl"): + os.rename( + os.path.join(FLAGS.dest_dir, fname), + os.path.join(FLAGS.dest_dir, fname[:-4] + f"-{label}.whl"), + ) + else: + print( + f"{_RED}=== PEP 817 variant: no nv/cu inputs detected; " + f"wheel emitted unlabeled{_RESET}", + file=sys.stderr, + ) + print(f"=== Output wheel file is in: {FLAGS.dest_dir}") touch(os.path.join(FLAGS.dest_dir, "stamp.whl"))