Skip to content

Commit e142ed7

Browse files
trxcllnttrxcllnt
authored
[FEA] Add GH release workflow (#395)
* add workflow action to build single-arch packages and create a github release * make C++ bindings lib names consistent by renaming node_ -> rapidsai_ * make packages image smaller by only copying the built tgz's into the final build stage * extract and publish the rapidsai_<name>.node files from the built tgz's * build ptx and sass
1 parent cadcefb commit e142ed7

58 files changed

Lines changed: 518 additions & 602 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/build-and-publish-image/action.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ runs:
100100
push: ${{ inputs.push }}
101101
file: ${{ inputs.file }}
102102
tags: ${{ inputs.tags }}
103+
load: ${{ inputs.push == 'false' }}
103104
platforms: ${{ inputs.platforms }}
104105
build-args: ${{ inputs.build-args }}
105106
labels: |

‎.github/workflows/main.pr.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
fail-fast: true
1414
matrix:
1515
CUDA: [11.6.2]
16-
LINUX: [ubuntu20.04]
16+
LINUX: ["ubuntu20.04"]
1717
env:
1818
NODE: 18.2.0
1919
RAPIDS: 22.02.00

‎.github/workflows/merge.pr.yml‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ jobs:
2323
fail-fast: true
2424
matrix:
2525
CUDA: ["11.6.2"]
26-
LINUX: [ubuntu20.04]
26+
LINUX: ["ubuntu20.04"]
2727
steps:
2828
- name: Checkout
2929
uses: actions/checkout@v2
@@ -71,7 +71,7 @@ jobs:
7171
fail-fast: true
7272
matrix:
7373
CUDA: ["11.6.2"]
74-
LINUX: [ubuntu20.04]
74+
LINUX: ["ubuntu20.04"]
7575
steps:
7676
- name: Checkout
7777
uses: actions/checkout@v2
@@ -125,7 +125,7 @@ jobs:
125125
fail-fast: true
126126
matrix:
127127
CUDA: ["11.6.2"]
128-
LINUX: [ubuntu20.04]
128+
LINUX: ["ubuntu20.04"]
129129
steps:
130130
- name: Checkout
131131
uses: actions/checkout@v2
@@ -172,7 +172,7 @@ jobs:
172172
fail-fast: true
173173
matrix:
174174
CUDA: ["11.6.2"]
175-
LINUX: [ubuntu20.04]
175+
LINUX: ["ubuntu20.04"]
176176
steps:
177177
- name: Checkout
178178
uses: actions/checkout@v2
@@ -207,7 +207,8 @@ jobs:
207207
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-cudf
208208
build-args: |
209209
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
210-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
210+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
211+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
211212
212213
build-and-publish-runtime-cugraph-image:
213214
needs:
@@ -219,7 +220,7 @@ jobs:
219220
fail-fast: true
220221
matrix:
221222
CUDA: ["11.6.2"]
222-
LINUX: [ubuntu20.04]
223+
LINUX: ["ubuntu20.04"]
223224
steps:
224225
- name: Checkout
225226
uses: actions/checkout@v2
@@ -254,7 +255,8 @@ jobs:
254255
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-cugraph
255256
build-args: |
256257
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
257-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
258+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
259+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
258260
259261
build-and-publish-runtime-cuml-image:
260262
needs:
@@ -266,7 +268,7 @@ jobs:
266268
fail-fast: true
267269
matrix:
268270
CUDA: ["11.6.2"]
269-
LINUX: [ubuntu20.04]
271+
LINUX: ["ubuntu20.04"]
270272
steps:
271273
- name: Checkout
272274
uses: actions/checkout@v2
@@ -301,7 +303,8 @@ jobs:
301303
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-cuml
302304
build-args: |
303305
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
304-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
306+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
307+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
305308
306309
build-and-publish-runtime-cuspatial-image:
307310
needs:
@@ -313,7 +316,7 @@ jobs:
313316
fail-fast: true
314317
matrix:
315318
CUDA: ["11.6.2"]
316-
LINUX: [ubuntu20.04]
319+
LINUX: ["ubuntu20.04"]
317320
steps:
318321
- name: Checkout
319322
uses: actions/checkout@v2
@@ -348,7 +351,8 @@ jobs:
348351
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-cuspatial
349352
build-args: |
350353
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
351-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
354+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
355+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
352356
353357
build-and-publish-runtime-glfw-image:
354358
needs:
@@ -360,7 +364,7 @@ jobs:
360364
fail-fast: true
361365
matrix:
362366
CUDA: ["11.6.2"]
363-
LINUX: [ubuntu20.04]
367+
LINUX: ["ubuntu20.04"]
364368
steps:
365369
- name: Checkout
366370
uses: actions/checkout@v2
@@ -395,7 +399,8 @@ jobs:
395399
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-glfw
396400
build-args: |
397401
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
398-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
402+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
403+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
399404
400405
build-and-publish-runtime-sql-image:
401406
needs:
@@ -407,7 +412,7 @@ jobs:
407412
fail-fast: true
408413
matrix:
409414
CUDA: ["11.6.2"]
410-
LINUX: [ubuntu20.04]
415+
LINUX: ["ubuntu20.04"]
411416
steps:
412417
- name: Checkout
413418
uses: actions/checkout@v2
@@ -442,7 +447,8 @@ jobs:
442447
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-sql
443448
build-args: |
444449
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
445-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
450+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
451+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
446452
447453
build-and-publish-runtime-main-image:
448454
needs:
@@ -454,7 +460,7 @@ jobs:
454460
fail-fast: true
455461
matrix:
456462
CUDA: ["11.6.2"]
457-
LINUX: [ubuntu20.04]
463+
LINUX: ["ubuntu20.04"]
458464
steps:
459465
- name: Checkout
460466
uses: actions/checkout@v2
@@ -489,7 +495,8 @@ jobs:
489495
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main
490496
build-args: |
491497
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
492-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
498+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
499+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
493500
494501
build-and-publish-runtime-demo-image:
495502
needs:
@@ -501,7 +508,7 @@ jobs:
501508
fail-fast: true
502509
matrix:
503510
CUDA: ["11.6.2"]
504-
LINUX: [ubuntu20.04]
511+
LINUX: ["ubuntu20.04"]
505512
steps:
506513
- name: Checkout
507514
uses: actions/checkout@v2
@@ -536,7 +543,8 @@ jobs:
536543
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-demo
537544
build-args: |
538545
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base"
539-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
546+
"BUILD_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"
547+
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
540548
541549
build-and-publish-runtime-notebook-image:
542550
needs:
@@ -547,7 +555,7 @@ jobs:
547555
fail-fast: true
548556
matrix:
549557
CUDA: ["11.6.2"]
550-
LINUX: [ubuntu20.04]
558+
LINUX: ["ubuntu20.04"]
551559
steps:
552560
- name: Checkout
553561
uses: actions/checkout@v2
@@ -583,4 +591,3 @@ jobs:
583591
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-notebook
584592
build-args: |
585593
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-demo"
586-
"DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages"

‎.github/workflows/release.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Create GitHub release
2+
3+
env:
4+
NODE: "18.2.0"
5+
RAPIDS: "22.02.00"
6+
VERSION: "0.0.1"
7+
REPOSITORY: "ghcr.io/rapidsai/node"
8+
9+
concurrency:
10+
group: create_github_release
11+
cancel-in-progress: true
12+
13+
permissions:
14+
contents: write
15+
16+
jobs:
17+
18+
build-extract-and-release-packages:
19+
name: Build, extract, and release packages
20+
runs-on: ubuntu-20.04
21+
strategy:
22+
fail-fast: true
23+
matrix:
24+
CUDA: ["11.6.2"]
25+
LINUX: ["ubuntu20.04"]
26+
GPU_ARCH: [60, 70, 80, 86, ALL]
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v2
30+
with:
31+
lfs: true
32+
fetch-depth: 2
33+
34+
- name: Checkout LFS
35+
run: git lfs checkout
36+
37+
- name: Build packages for GPU arch ${{ matrix.GPU_ARCH }}
38+
uses: ./.github/actions/build-and-publish-image
39+
with:
40+
registry-url: ghcr.io
41+
registry-username: ${{ github.repository_owner }}
42+
registry-password: ${{ github.token }}
43+
pull: true
44+
push: false
45+
AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
46+
AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
47+
RAPIDSAI_GITHUB_ACCESS_TOKEN: "${{ secrets.RAPIDSAI_GITHUB_ACCESS_TOKEN }}"
48+
file: dev/dockerfiles/devel/package.Dockerfile
49+
tags: |
50+
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages-${{ matrix.GPU_ARCH }}
51+
build-args: |
52+
"CUDAARCHS=${{ matrix.GPU_ARCH }}-real"
53+
"PARALLEL_LEVEL=1"
54+
"SCCACHE_IDLE_TIMEOUT=32768"
55+
"SCCACHE_REGION=us-west-2"
56+
"SCCACHE_BUCKET=node-rapids-sccache"
57+
"RAPIDS_VERSION=${{ env.RAPIDS }}"
58+
"FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-main"
59+
60+
- name: Extract packages for GPU arch ${{ matrix.GPU_ARCH }}
61+
run: |
62+
mkdir -p build
63+
docker run --rm -v "$PWD/build:/out" \
64+
${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages-${{ matrix.GPU_ARCH }} \
65+
sh -c "cp /opt/rapids/*.tgz /out/"
66+
GPU_ARCH="$(echo "${{ matrix.GPU_ARCH }}" | tr '[:upper:]' '[:lower:]')"
67+
for x in cuda cudf cugraph cuml cuspatial io rmm sql; do
68+
tar -C build -zf build/rapidsai-$x-*.tgz --strip-components=3 -x package/build/Release/rapidsai_$x.node;
69+
mv build/rapidsai_$x.node build/rapidsai_$x-$GPU_ARCH-${{ env.RAPIDS }}.node;
70+
done;
71+
rm -rf build/*.tgz;
72+
73+
- name: Create GitHub release
74+
uses: softprops/action-gh-release@v1
75+
env:
76+
GITHUB_REPOSITORY: ${{ github.repository }}
77+
with:
78+
body: ""
79+
draft: true
80+
tag_name: "v${{ env.RAPIDS }}"
81+
token: ${{ secrets.RAPIDSAI_GITHUB_ACCESS_TOKEN }}
82+
files: |
83+
build/*.node

‎dev/dockerfiles/devel/package.Dockerfile‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@
22

33
ARG FROM_IMAGE
44

5-
FROM ${FROM_IMAGE}
5+
FROM ${FROM_IMAGE} as build
66

77
WORKDIR /opt/rapids/node
88

99
ENV NVIDIA_DRIVER_CAPABILITIES all
1010

1111
ARG CUDAARCHS=ALL
1212
ARG PARALLEL_LEVEL
13+
ARG NVCC_APPEND_FLAGS
1314
ARG RAPIDS_VERSION
1415
ARG SCCACHE_REGION
1516
ARG SCCACHE_BUCKET
@@ -41,6 +42,7 @@ RUN --mount=type=secret,id=sccache_credentials \
4142
&& bash -c 'echo -e "\
4243
CUDAARCHS=$CUDAARCHS\n\
4344
PARALLEL_LEVEL=$PARALLEL_LEVEL\n\
45+
NVCC_APPEND_FLAGS=$NVCC_APPEND_FLAGS\n\
4446
RAPIDS_VERSION=$RAPIDS_VERSION\n\
4547
SCCACHE_REGION=$SCCACHE_REGION\n\
4648
SCCACHE_BUCKET=$SCCACHE_BUCKET\n\
@@ -49,11 +51,9 @@ SCCACHE_IDLE_TIMEOUT=$SCCACHE_IDLE_TIMEOUT\n\
4951
&& yarn --pure-lockfile --network-timeout 1000000 \
5052
&& yarn build \
5153
&& yarn dev:npm:pack \
52-
&& yarn cache clean \
53-
&& cp build/*.tgz ../ \
54-
&& cd .. && rm -rf node \
55-
&& chown -R rapids:rapids .
54+
&& chown rapids:rapids build/*.tgz \
55+
&& mv build/*.tgz ../
5656

57-
USER rapids
57+
FROM alpine:latest
5858

59-
WORKDIR /home/node
59+
COPY --from=build /opt/rapids/*.tgz /opt/rapids/

‎dev/dockerfiles/runtime/cudf.Dockerfile‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,27 @@
11
ARG FROM_IMAGE
2+
ARG BUILD_IMAGE
23
ARG DEVEL_IMAGE
34

5+
FROM ${BUILD_IMAGE} as build
46
FROM ${DEVEL_IMAGE} as devel
57

68
WORKDIR /home/node
79

8-
RUN cp \
9-
/opt/rapids/rapidsai-core-*.tgz \
10-
/opt/rapids/rapidsai-cuda-*.tgz \
11-
/opt/rapids/rapidsai-rmm-*.tgz \
12-
/opt/rapids/rapidsai-cudf-*.tgz \
13-
. \
14-
&& npm install --production --omit dev --omit peer --omit optional --legacy-peer-deps --force *.tgz
10+
RUN --mount=type=bind,from=build,source=/opt/rapids/,target=/tmp/rapids/ \
11+
npm install --omit=dev --omit=peer --omit=optional --legacy-peer-deps --force \
12+
/tmp/rapids/rapidsai-core-*.tgz \
13+
/tmp/rapids/rapidsai-cuda-*.tgz \
14+
/tmp/rapids/rapidsai-rmm-*.tgz \
15+
/tmp/rapids/rapidsai-cudf-*.tgz ;
1516

1617
FROM ${FROM_IMAGE}
1718

1819
SHELL ["/bin/bash", "-c"]
1920

20-
COPY --from=devel --chown=node:node /home/node/node_modules/ /home/node/node_modules/
21-
2221
WORKDIR /home/node
2322

23+
COPY --from=devel --chown=node:node /home/node/node_modules .
24+
2425
SHELL ["/bin/bash", "-l"]
2526

2627
CMD ["node"]

0 commit comments

Comments
 (0)