Skip to content

Commit a9064fe

Browse files
jameslambjameslamb
authored
enforce a floor on libnvjitlink, build wheels with CUDA 13.0.x, test wheels against mix of CTK versions (rapidsai#21671)
Contributes to rapidsai/build-planning#257 * builds CUDA 13 wheels with the 13.0 CTK * ensures wheels ship with a runtime dependency of `nvidia-nvjitlink>={whatever-minor-version-they-were-built-against}` Contributes to rapidsai/build-planning#256 * updates wheel tests to cover a range of CTK versions (we previously, accidentally, were only testing the latest 12.x and 13.x) Other changes * ensures conda packages also take on floors of `libnvjitlink>={whatever-version-they-were-built-against}` ## Notes for Reviewers ### How I tested this Used wheels from similar PRs from RAPIDS dependencies, at build and test time: * rapidsai/kvikio#942 * rapidsai/rmm#2270 * rapidsai/ucxx#604 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Bradley Dice (https://github.com/bdice) URL: rapidsai#21671
1 parent b13ada2 commit a9064fe

11 files changed

Lines changed: 180 additions & 29 deletions

File tree

‎ci/test_cudf_polars_polars_tests.sh‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,24 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF
1313
LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
1414
PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")")
1515

16+
# generate constraints (possibly pinning to oldest support versions of dependencies)
17+
rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}"
18+
1619
rapids-logger "Install libcudf, pylibcudf and cudf_polars"
20+
21+
# notes:
22+
#
23+
# * echo to expand wildcard before adding `[test]` requires for pip
24+
# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
25+
# that environment variable is ignored if any other --constraint are passed via the CLI
26+
#
1727
rapids-pip-retry install \
1828
-v \
29+
--constraint "${PIP_CONSTRAINT}" \
1930
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \
2031
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
2132
"$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)"
2233

23-
2434
TAG=$(python -c 'import polars; print(f"py-{polars.__version__}")')
2535
rapids-logger "Clone polars to ${TAG}"
2636
git clone https://github.com/pola-rs/polars.git --branch "${TAG}" --depth 1
@@ -55,7 +65,17 @@ sed -i 's/^pandas-stubs/pandas-stubs<3/' polars/py-polars/requirements-dev.txt
5565
# existing dependency to rewrite.
5666
echo "pyparsing>=3.0.0,<3.3.0" >> polars/py-polars/requirements-dev.txt
5767

58-
rapids-pip-retry install -r polars/py-polars/requirements-dev.txt -r polars/py-polars/requirements-ci.txt
68+
# notes:
69+
#
70+
# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
71+
# that environment variable is ignored if any other --constraint are passed via the CLI
72+
#
73+
rapids-pip-retry install \
74+
-v \
75+
--prefer-binary \
76+
--constraint "${PIP_CONSTRAINT}" \
77+
-r polars/py-polars/requirements-dev.txt \
78+
-r polars/py-polars/requirements-ci.txt
5979

6080
# shellcheck disable=SC2317
6181
function set_exitcode()

‎ci/test_cudf_polars_with_rapidsmpf.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,18 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel
1515

1616
rapids-logger "Installing cudf_polars and its dependencies (including rapidsmpf)"
1717

18+
# generate constraints (possibly pinning to oldest support versions of dependencies)
19+
rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}"
20+
21+
# notes:
22+
#
23+
# * echo to expand wildcard before adding `[test]` requires for pip
24+
# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
25+
# that environment variable is ignored if any other --constraint are passed via the CLI
26+
#
1827
rapids-pip-retry install \
1928
-v \
29+
--prefer-binary \
2030
--constraint "${PIP_CONSTRAINT}" \
2131
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,rapidsmpf]" \
2232
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \

‎ci/test_wheel_cudf.sh‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel
1616
rapids-logger "Install pylibcudf and its basic dependencies in a virtual environment"
1717

1818
# generate constraints (possibly pinning to oldest support versions of dependencies)
19-
rapids-generate-pip-constraints py_test_cudf ./constraints.txt
19+
rapids-generate-pip-constraints py_test_cudf "${PIP_CONSTRAINT}"
2020

2121
RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}
2222
RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/
@@ -25,9 +25,16 @@ mkdir -p "${RAPIDS_TESTS_DIR}"
2525
# To test pylibcudf without its optional dependencies, we create a virtual environment
2626
python -m venv env
2727
. env/bin/activate
28+
29+
# notes:
30+
#
31+
# * echo to expand wildcard before adding `[test]` requires for pip
32+
# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
33+
# that environment variable is ignored if any other --constraint are passed via the CLI
34+
#
2835
rapids-pip-retry install \
2936
-v \
30-
--constraint ./constraints.txt \
37+
--prefer-binary \
3138
--constraint "${PIP_CONSTRAINT}" \
3239
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
3340
"$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]"
@@ -48,12 +55,12 @@ rapids-logger "Install cudf, pylibcudf, and test requirements"
4855
# notes:
4956
#
5057
# * echo to expand wildcard before adding `[test]` requires for pip
51-
# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is
52-
# ignored if any other --constraint are passed via the CLI
58+
# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
59+
# that environment variable is ignored if any other --constraint are passed via the CLI
5360
#
5461
rapids-pip-retry install \
5562
-v \
56-
--constraint ./constraints.txt \
63+
--prefer-binary \
5764
--constraint "${PIP_CONSTRAINT}" \
5865
"$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \
5966
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \

‎ci/test_wheel_cudf_polars.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,17 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel
1818
rapids-logger "Installing cudf_polars and its dependencies"
1919

2020
# generate constraints (possibly pinning to oldest support versions of dependencies)
21-
rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt
21+
rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}"
2222

2323
# notes:
2424
#
25-
# * echo to expand wildcard before adding `[test,experimental]` requires for pip
26-
# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is
27-
# ignored if any other --constraint are passed via the CLI
25+
# * echo to expand wildcard before adding `[test]` requires for pip
26+
# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
27+
# that environment variable is ignored if any other --constraint are passed via the CLI
2828
#
2929
rapids-pip-retry install \
3030
-v \
31-
--constraint ./constraints.txt \
31+
--prefer-binary \
3232
--constraint "${PIP_CONSTRAINT}" \
3333
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental]" \
3434
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \

‎ci/test_wheel_dask_cudf.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,17 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel
1717
rapids-logger "Install dask_cudf, cudf, pylibcudf, and test requirements"
1818

1919
# generate constraints (possibly pinning to oldest support versions of dependencies)
20-
rapids-generate-pip-constraints py_test_dask_cudf ./constraints.txt
20+
rapids-generate-pip-constraints py_test_dask_cudf "${PIP_CONSTRAINT}"
2121

2222
# notes:
2323
#
2424
# * echo to expand wildcard before adding `[test]` requires for pip
25-
# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is
26-
# ignored if any other --constraint are passed via the CLI
25+
# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
26+
# that environment variable is ignored if any other --constraint are passed via the CLI
2727
#
2828
rapids-pip-retry install \
2929
-v \
30-
--constraint ./constraints.txt \
30+
--prefer-binary \
3131
--constraint "${PIP_CONSTRAINT}" \
3232
"$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
3333
"$(echo "${DASK_CUDF_WHEELHOUSE}"/dask_cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \

‎conda/recipes/libcudf/recipe.yaml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,6 @@ outputs:
129129
run:
130130
- ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }}
131131
- cuda-nvrtc
132-
- libnvjitlink >=${{ cuda_version }}
133132
- if: linux and x86_64
134133
then:
135134
- libcufile
@@ -154,7 +153,6 @@ outputs:
154153
- libcufile
155154
- libcurand
156155
- libkvikio
157-
- libnvjitlink
158156
- librdkafka
159157
- libzlib
160158
- librmm

‎dependencies.yaml‎

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ files:
204204
table: project.optional-dependencies
205205
key: test
206206
includes:
207+
- cuda_version
207208
- test_python_common
208209
- test_python_cudf_common
209210
- test_python_cudf
@@ -352,6 +353,7 @@ files:
352353
table: project.optional-dependencies
353354
key: test
354355
includes:
356+
- cuda_version
355357
- depends_on_dask_cuda
356358
- numpy_run
357359
- test_python_common
@@ -390,6 +392,7 @@ files:
390392
table: project.optional-dependencies
391393
key: test
392394
includes:
395+
- cuda_version
393396
- depends_on_dask_cuda
394397
- test_python_common
395398
- test_python_cudf_common
@@ -563,6 +566,8 @@ dependencies:
563566
- output_types: [requirements, pyproject]
564567
packages:
565568
- pyarrow>=19.0.0
569+
# 'cuda_version' intentionally does not contain fallback entries... we want
570+
# a loud error if an unsupported 'cuda' value is passed
566571
cuda_version:
567572
specific:
568573
- output_types: conda
@@ -591,6 +596,51 @@ dependencies:
591596
cuda: "13.1"
592597
packages:
593598
- cuda-version=13.1
599+
- output_types: requirements
600+
matrices:
601+
# if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels
602+
# (e.g. for DLFW and pip devcontainers)
603+
- matrix:
604+
use_cuda_wheels: "false"
605+
packages:
606+
- matrix:
607+
arch: aarch64
608+
cuda: "12.2"
609+
use_cuda_wheels: "true"
610+
packages:
611+
# some components (like nvidia-cublas-cu12 and nvidia-cuda-nvcc-cu12) didn't have
612+
# aarch64 wheels until CTK 12.3, so allow a slightly looser bound here
613+
- cuda-toolkit>=12.2,<12.4
614+
- matrix:
615+
cuda: "12.2"
616+
use_cuda_wheels: "true"
617+
packages:
618+
- cuda-toolkit==12.2.*
619+
- matrix:
620+
cuda: "12.5"
621+
use_cuda_wheels: "true"
622+
packages:
623+
- cuda-toolkit==12.5.*
624+
- matrix:
625+
cuda: "12.8"
626+
use_cuda_wheels: "true"
627+
packages:
628+
- cuda-toolkit==12.8.*
629+
- matrix:
630+
cuda: "12.9"
631+
use_cuda_wheels: "true"
632+
packages:
633+
- cuda-toolkit==12.9.*
634+
- matrix:
635+
cuda: "13.0"
636+
use_cuda_wheels: "true"
637+
packages:
638+
- cuda-toolkit==13.0.*
639+
- matrix:
640+
cuda: "13.1"
641+
use_cuda_wheels: "true"
642+
packages:
643+
- cuda-toolkit==13.1.*
594644
cuda:
595645
common:
596646
- output_types: [conda]
@@ -604,14 +654,30 @@ dependencies:
604654
specific:
605655
- output_types: [requirements, pyproject]
606656
matrices:
657+
# if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels
658+
# (e.g. for DLFW and pip devcontainers)
607659
- matrix:
608-
cuda: "12.*"
660+
use_cuda_wheels: "false"
609661
packages:
610-
- nvidia-nvjitlink-cu12>=12.9
611-
# fallback to CUDA 13 versions if 'cuda' is '13.*' or not provided
662+
# We always want nvJitLink >= whatever minor version was available in the build environment
663+
# nvJitLink tends to follow the CTK's minor versions, but check at https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html
664+
#
665+
# ref: https://github.com/rapidsai/cudf/pull/20873
666+
- matrix:
667+
cuda: "12.9"
668+
use_cuda_wheels: "true"
669+
packages:
670+
- nvidia-nvjitlink-cu12>=12.9,<13
671+
- matrix:
672+
cuda: "13.*"
673+
use_cuda_wheels: "true"
674+
packages:
675+
- &nvjitlink_cu13 nvidia-nvjitlink>=13.0,<14
676+
# if no matching matrix selectors passed, list the CUDA 13 requirement
677+
# (just as a source of documentation, as this populates pyproject.toml in source control)
612678
- matrix:
613679
packages:
614-
- nvidia-nvjitlink>=13.0
680+
- *nvjitlink_cu13
615681
develop:
616682
common:
617683
- output_types: [conda, requirements]
@@ -725,14 +791,21 @@ dependencies:
725791
specific:
726792
- output_types: [requirements, pyproject]
727793
matrices:
794+
# if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels
795+
# (e.g. for DLFW and pip devcontainers)
796+
- matrix:
797+
use_cuda_wheels: "false"
798+
packages:
728799
- matrix:
729800
cuda: "12.*"
730801
cuda_suffixed: "true"
802+
use_cuda_wheels: "true"
731803
packages:
732804
- cuda-toolkit[nvcc,nvrtc]==12.*
733805
- matrix:
734806
cuda: "13.*"
735807
cuda_suffixed: "true"
808+
use_cuda_wheels: "true"
736809
packages:
737810
- cuda-toolkit[nvcc,nvrtc]==13.*
738811
- {matrix: null, packages: []}

‎python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,14 +138,12 @@ dependencies:
138138
- output_types: conda
139139
packages:
140140
- pre-commit
141+
# 'cuda_version' intentionally does not contain fallback entries... we want
142+
# a loud error if an unsupported 'cuda' value is passed
141143
cuda_version:
142144
specific:
143145
- output_types: conda
144146
matrices:
145-
- matrix:
146-
cuda: "12.0"
147-
packages:
148-
- cuda-version=12.0
149147
- matrix:
150148
cuda: "12.2"
151149
packages:
@@ -170,6 +168,51 @@ dependencies:
170168
cuda: "13.1"
171169
packages:
172170
- cuda-version=13.1
171+
- output_types: requirements
172+
matrices:
173+
# if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels
174+
# (e.g. for DLFW and pip devcontainers)
175+
- matrix:
176+
use_cuda_wheels: "false"
177+
packages:
178+
- matrix:
179+
arch: aarch64
180+
cuda: "12.2"
181+
use_cuda_wheels: "true"
182+
packages:
183+
# some components (like nvidia-cublas-cu12 and nvidia-cuda-nvcc-cu12) didn't have
184+
# aarch64 wheels until CTK 12.3, so allow a slightly looser bound here
185+
- cuda-toolkit>=12.2,<12.4
186+
- matrix:
187+
cuda: "12.2"
188+
use_cuda_wheels: "true"
189+
packages:
190+
- cuda-toolkit==12.2.*
191+
- matrix:
192+
cuda: "12.5"
193+
use_cuda_wheels: "true"
194+
packages:
195+
- cuda-toolkit==12.5.*
196+
- matrix:
197+
cuda: "12.8"
198+
use_cuda_wheels: "true"
199+
packages:
200+
- cuda-toolkit==12.8.*
201+
- matrix:
202+
cuda: "12.9"
203+
use_cuda_wheels: "true"
204+
packages:
205+
- cuda-toolkit==12.9.*
206+
- matrix:
207+
cuda: "13.0"
208+
use_cuda_wheels: "true"
209+
packages:
210+
- cuda-toolkit==13.0.*
211+
- matrix:
212+
cuda: "13.1"
213+
use_cuda_wheels: "true"
214+
packages:
215+
- cuda-toolkit==13.1.*
173216
py_version:
174217
specific:
175218
- output_types: conda

‎python/cudf/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ xfail_strict = true
121121
[tool.rapids-build-backend]
122122
build-backend = "scikit_build_core.build"
123123
dependencies-file = "../../dependencies.yaml"
124-
matrix-entry = "cuda_suffixed=true"
124+
matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true"
125125
requires = [
126126
"cmake>=3.30.4",
127127
"cython>=3.2.2",

‎python/libcudf/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,6 @@ requires = [
7373
"librmm==26.4.*,>=0.0.0a0",
7474
"ninja",
7575
"nvidia-libnvcomp==5.1.0.21",
76-
"nvidia-nvjitlink>=13.0",
76+
"nvidia-nvjitlink>=13.0,<14",
7777
"rapids-logger==0.2.*,>=0.0.0a0",
7878
] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

0 commit comments

Comments
 (0)