Skip to content

Commit a0854c0

Browse files
trxcllnttrxcllnt
committed
enable sccache-dist build cluster
1 parent 261292b commit a0854c0

6 files changed

Lines changed: 92 additions & 52 deletions

File tree

‎.devcontainer/llvm20-cuda12.0/devcontainer.json‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,29 @@
55

66
"initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,cache,config}"],
77

8+
"postCreateCommand": [
9+
"/bin/bash",
10+
"-c",
11+
"if test -z \"${DISABLE_SCCACHE:+x}\"; then echo \"export SCCACHE_DIST_URL='https://$(dpkg --print-architecture).$(uname -s | tr '[:upper:]' '[:lower:]').sccache.rapids.nvidia.com'\" >> /home/coder/.bashrc; fi"
12+
],
13+
814
"containerEnv": {
915
"AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
1016
"DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "true",
1117
"HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
1218
"SCCACHE_BUCKET": "rapids-sccache-devs",
13-
"SCCACHE_DIST_AUTH_TOKEN_VAR": "${localEnv:SCCACHE_DIST_AUTH_TOKEN_VAR}",
14-
"SCCACHE_DIST_CONNECT_TIMEOUT": "${localEnv:SCCACHE_DIST_CONNECT_TIMEOUT:30}",
15-
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "${localEnv:SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE:true}",
16-
"SCCACHE_DIST_KEEPALIVE_ENABLED": "${localEnv:SCCACHE_DIST_KEEPALIVE_ENABLED:true}",
17-
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "${localEnv:SCCACHE_DIST_KEEPALIVE_INTERVAL:20}",
18-
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "${localEnv:SCCACHE_DIST_KEEPALIVE_TIMEOUT:600}",
19-
"SCCACHE_DIST_MAX_RETRIES": "${localEnv:SCCACHE_DIST_MAX_RETRIES:4}",
20-
"SCCACHE_DIST_REQUEST_TIMEOUT": "${localEnv:SCCACHE_DIST_REQUEST_TIMEOUT:7140}",
21-
"SCCACHE_DIST_TOKEN": "${localEnv:SCCACHE_DIST_TOKEN}",
19+
"SCCACHE_DIST_CONNECT_TIMEOUT": "30",
20+
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "true",
21+
"SCCACHE_DIST_KEEPALIVE_ENABLED": "true",
22+
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "20",
23+
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "600",
24+
"SCCACHE_DIST_MAX_RETRIES": "4",
25+
"SCCACHE_DIST_REQUEST_TIMEOUT": "7140",
2226
"SCCACHE_ERROR_LOG": "/home/coder/stdexec/.cache/sccache.log",
2327
"SCCACHE_REGION": "us-east-2",
2428
"SCCACHE_S3_KEY_PREFIX": "nvidia-stdexec-dev",
2529
"SCCACHE_SERVER_PORT": "4225",
26-
"SCCACHE_SERVER_LOG": "${localEnv:SCCACHE_SERVER_LOG:sccache=debug}"
30+
"SCCACHE_SERVER_LOG": "sccache=debug"
2731
},
2832

2933
"workspaceFolder": "/home/coder/stdexec",

‎.devcontainer/llvm20-cuda12.9/devcontainer.json‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,29 @@
55

66
"initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,cache,config}"],
77

8+
"postCreateCommand": [
9+
"/bin/bash",
10+
"-c",
11+
"if test -z \"${DISABLE_SCCACHE:+x}\"; then echo \"export SCCACHE_DIST_URL='https://$(dpkg --print-architecture).$(uname -s | tr '[:upper:]' '[:lower:]').sccache.rapids.nvidia.com'\" >> /home/coder/.bashrc; fi"
12+
],
13+
814
"containerEnv": {
915
"AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
1016
"DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "true",
1117
"HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
1218
"SCCACHE_BUCKET": "rapids-sccache-devs",
13-
"SCCACHE_DIST_AUTH_TOKEN_VAR": "${localEnv:SCCACHE_DIST_AUTH_TOKEN_VAR}",
14-
"SCCACHE_DIST_CONNECT_TIMEOUT": "${localEnv:SCCACHE_DIST_CONNECT_TIMEOUT:30}",
15-
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "${localEnv:SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE:true}",
16-
"SCCACHE_DIST_KEEPALIVE_ENABLED": "${localEnv:SCCACHE_DIST_KEEPALIVE_ENABLED:true}",
17-
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "${localEnv:SCCACHE_DIST_KEEPALIVE_INTERVAL:20}",
18-
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "${localEnv:SCCACHE_DIST_KEEPALIVE_TIMEOUT:600}",
19-
"SCCACHE_DIST_MAX_RETRIES": "${localEnv:SCCACHE_DIST_MAX_RETRIES:4}",
20-
"SCCACHE_DIST_REQUEST_TIMEOUT": "${localEnv:SCCACHE_DIST_REQUEST_TIMEOUT:7140}",
21-
"SCCACHE_DIST_TOKEN": "${localEnv:SCCACHE_DIST_TOKEN}",
19+
"SCCACHE_DIST_CONNECT_TIMEOUT": "30",
20+
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "true",
21+
"SCCACHE_DIST_KEEPALIVE_ENABLED": "true",
22+
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "20",
23+
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "600",
24+
"SCCACHE_DIST_MAX_RETRIES": "4",
25+
"SCCACHE_DIST_REQUEST_TIMEOUT": "7140",
2226
"SCCACHE_ERROR_LOG": "/home/coder/stdexec/.cache/sccache.log",
2327
"SCCACHE_REGION": "us-east-2",
2428
"SCCACHE_S3_KEY_PREFIX": "nvidia-stdexec-dev",
2529
"SCCACHE_SERVER_PORT": "4225",
26-
"SCCACHE_SERVER_LOG": "${localEnv:SCCACHE_SERVER_LOG:sccache=debug}"
30+
"SCCACHE_SERVER_LOG": "sccache=debug"
2731
},
2832

2933
"workspaceFolder": "/home/coder/stdexec",

‎.devcontainer/llvm20-cuda13.0/devcontainer.json‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,29 @@
55

66
"initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,cache,config}"],
77

8+
"postCreateCommand": [
9+
"/bin/bash",
10+
"-c",
11+
"if test -z \"${DISABLE_SCCACHE:+x}\"; then echo \"export SCCACHE_DIST_URL='https://$(dpkg --print-architecture).$(uname -s | tr '[:upper:]' '[:lower:]').sccache.rapids.nvidia.com'\" >> /home/coder/.bashrc; fi"
12+
],
13+
814
"containerEnv": {
915
"AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
1016
"DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "true",
1117
"HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
1218
"SCCACHE_BUCKET": "rapids-sccache-devs",
13-
"SCCACHE_DIST_AUTH_TOKEN_VAR": "${localEnv:SCCACHE_DIST_AUTH_TOKEN_VAR}",
14-
"SCCACHE_DIST_CONNECT_TIMEOUT": "${localEnv:SCCACHE_DIST_CONNECT_TIMEOUT:30}",
15-
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "${localEnv:SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE:true}",
16-
"SCCACHE_DIST_KEEPALIVE_ENABLED": "${localEnv:SCCACHE_DIST_KEEPALIVE_ENABLED:true}",
17-
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "${localEnv:SCCACHE_DIST_KEEPALIVE_INTERVAL:20}",
18-
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "${localEnv:SCCACHE_DIST_KEEPALIVE_TIMEOUT:600}",
19-
"SCCACHE_DIST_MAX_RETRIES": "${localEnv:SCCACHE_DIST_MAX_RETRIES:4}",
20-
"SCCACHE_DIST_REQUEST_TIMEOUT": "${localEnv:SCCACHE_DIST_REQUEST_TIMEOUT:7140}",
21-
"SCCACHE_DIST_TOKEN": "${localEnv:SCCACHE_DIST_TOKEN}",
19+
"SCCACHE_DIST_CONNECT_TIMEOUT": "30",
20+
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "true",
21+
"SCCACHE_DIST_KEEPALIVE_ENABLED": "true",
22+
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "20",
23+
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "600",
24+
"SCCACHE_DIST_MAX_RETRIES": "4",
25+
"SCCACHE_DIST_REQUEST_TIMEOUT": "7140",
2226
"SCCACHE_ERROR_LOG": "/home/coder/stdexec/.cache/sccache.log",
2327
"SCCACHE_REGION": "us-east-2",
2428
"SCCACHE_S3_KEY_PREFIX": "nvidia-stdexec-dev",
2529
"SCCACHE_SERVER_PORT": "4225",
26-
"SCCACHE_SERVER_LOG": "${localEnv:SCCACHE_SERVER_LOG:sccache=debug}"
30+
"SCCACHE_SERVER_LOG": "sccache=debug"
2731
},
2832

2933
"workspaceFolder": "/home/coder/stdexec",

‎.devcontainer/nvhpc25.7/devcontainer.json‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,29 @@
55

66
"initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,cache,config}"],
77

8+
"postCreateCommand": [
9+
"/bin/bash",
10+
"-c",
11+
"if test -z \"${DISABLE_SCCACHE:+x}\"; then echo \"export SCCACHE_DIST_URL='https://$(dpkg --print-architecture).$(uname -s | tr '[:upper:]' '[:lower:]').sccache.rapids.nvidia.com'\" >> /home/coder/.bashrc; fi"
12+
],
13+
814
"containerEnv": {
915
"AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
1016
"DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "true",
1117
"HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
1218
"SCCACHE_BUCKET": "rapids-sccache-devs",
13-
"SCCACHE_DIST_AUTH_TOKEN_VAR": "${localEnv:SCCACHE_DIST_AUTH_TOKEN_VAR}",
14-
"SCCACHE_DIST_CONNECT_TIMEOUT": "${localEnv:SCCACHE_DIST_CONNECT_TIMEOUT:30}",
15-
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "${localEnv:SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE:true}",
16-
"SCCACHE_DIST_KEEPALIVE_ENABLED": "${localEnv:SCCACHE_DIST_KEEPALIVE_ENABLED:true}",
17-
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "${localEnv:SCCACHE_DIST_KEEPALIVE_INTERVAL:20}",
18-
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "${localEnv:SCCACHE_DIST_KEEPALIVE_TIMEOUT:600}",
19-
"SCCACHE_DIST_MAX_RETRIES": "${localEnv:SCCACHE_DIST_MAX_RETRIES:4}",
20-
"SCCACHE_DIST_REQUEST_TIMEOUT": "${localEnv:SCCACHE_DIST_REQUEST_TIMEOUT:7140}",
21-
"SCCACHE_DIST_TOKEN": "${localEnv:SCCACHE_DIST_TOKEN}",
19+
"SCCACHE_DIST_CONNECT_TIMEOUT": "30",
20+
"SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "true",
21+
"SCCACHE_DIST_KEEPALIVE_ENABLED": "true",
22+
"SCCACHE_DIST_KEEPALIVE_INTERVAL": "20",
23+
"SCCACHE_DIST_KEEPALIVE_TIMEOUT": "600",
24+
"SCCACHE_DIST_MAX_RETRIES": "4",
25+
"SCCACHE_DIST_REQUEST_TIMEOUT": "7140",
2226
"SCCACHE_ERROR_LOG": "/home/coder/stdexec/.cache/sccache.log",
2327
"SCCACHE_REGION": "us-east-2",
2428
"SCCACHE_S3_KEY_PREFIX": "nvidia-stdexec-dev",
2529
"SCCACHE_SERVER_PORT": "4225",
26-
"SCCACHE_SERVER_LOG": "${localEnv:SCCACHE_SERVER_LOG:sccache=debug}"
30+
"SCCACHE_SERVER_LOG": "sccache=debug"
2731
},
2832

2933
"workspaceFolder": "/home/coder/stdexec",

‎.github/workflows/ci.cpu.yml‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,23 @@ jobs:
3030
options: -u root
3131
image: rapidsai/devcontainers:25.10-cpp-${{ matrix.tag }}
3232
env:
33-
SCCACHE_REGION: "us-east-2"
33+
DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST: "true"
3434
SCCACHE_BUCKET: "rapids-sccache-devs"
35+
SCCACHE_DIST_AUTH_TOKEN_VAR: "SCCACHE_DIST_TOKEN"
36+
SCCACHE_DIST_CONNECT_TIMEOUT: "30"
37+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: "true"
38+
SCCACHE_DIST_KEEPALIVE_ENABLED: "true"
39+
SCCACHE_DIST_KEEPALIVE_INTERVAL: "20"
40+
SCCACHE_DIST_KEEPALIVE_TIMEOUT: "600"
41+
SCCACHE_DIST_MAX_RETRIES: "4"
42+
SCCACHE_DIST_REQUEST_TIMEOUT: "7140"
43+
SCCACHE_DIST_TOKEN: "${{ secrets.STDEXEC_BUILD_CLUSTER_SECRET }}"
44+
SCCACHE_DIST_URL: "https://${{ matrix.arch }}.linux.sccache.rapids.nvidia.com"
45+
SCCACHE_ERROR_LOG: "/home/coder/stdexec/.cache/sccache.log"
46+
SCCACHE_REGION: "us-east-2"
3547
SCCACHE_S3_KEY_PREFIX: "nvidia-stdexec-dev"
36-
SCCACHE_ERROR_LOG: "/home/coder/.cache/sccache.log"
37-
SCCACHE_SERVER_LOG: "sccache=debug"
3848
SCCACHE_SERVER_PORT: "4225"
49+
SCCACHE_SERVER_LOG: "sccache=debug"
3950
permissions:
4051
id-token: write # This is required for configure-aws-credentials
4152
contents: read # This is required for actions/checkout
@@ -78,8 +89,9 @@ jobs:
7889
run: |
7990
set -ex;
8091
81-
devcontainer-utils-install-sccache --repo rapidsai/sccache --version rapids;
82-
devcontainer-utils-start-sccache;
92+
sudo pkill -9 sccache || :;
93+
sudo rm -f .cache/*.log;
94+
devcontainer-utils-post-attach-command;
8395
8496
# Copy source folder into ~/stdexec
8597
cp -r "${GITHUB_WORKSPACE}"/stdexec ~/;
@@ -108,7 +120,7 @@ jobs:
108120
uses: actions/upload-artifact@v4
109121
with:
110122
name: sccache-client-logs-${{env.ARTIFACT_PREFIX}}-${{env.ARTIFACT_SUFFIX}}
111-
path: /home/coder/.cache/sccache*.log
123+
path: .cache/*.log
112124
compression-level: 9
113125

114126
ci-cpu:

‎.github/workflows/ci.gpu.yml‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,24 @@ jobs:
2929
options: -u root
3030
image: rapidsai/devcontainers:25.10-cpp-${{ matrix.tag }}
3131
env:
32+
DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST: "true"
3233
NVIDIA_VISIBLE_DEVICES: "${{ env.NVIDIA_VISIBLE_DEVICES }}"
33-
SCCACHE_REGION: "us-east-2"
3434
SCCACHE_BUCKET: "rapids-sccache-devs"
35+
SCCACHE_DIST_AUTH_TOKEN_VAR: "SCCACHE_DIST_TOKEN"
36+
SCCACHE_DIST_CONNECT_TIMEOUT: "30"
37+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: "true"
38+
SCCACHE_DIST_KEEPALIVE_ENABLED: "true"
39+
SCCACHE_DIST_KEEPALIVE_INTERVAL: "20"
40+
SCCACHE_DIST_KEEPALIVE_TIMEOUT: "600"
41+
SCCACHE_DIST_MAX_RETRIES: "4"
42+
SCCACHE_DIST_REQUEST_TIMEOUT: "7140"
43+
SCCACHE_DIST_TOKEN: "${{ secrets.STDEXEC_BUILD_CLUSTER_SECRET }}"
44+
SCCACHE_DIST_URL: "https://${{ matrix.arch }}.linux.sccache.rapids.nvidia.com"
45+
SCCACHE_ERROR_LOG: "/home/coder/stdexec/.cache/sccache.log"
46+
SCCACHE_REGION: "us-east-2"
3547
SCCACHE_S3_KEY_PREFIX: "nvidia-stdexec-dev"
36-
SCCACHE_ERROR_LOG: "/home/coder/.cache/sccache.log"
37-
SCCACHE_SERVER_LOG: "sccache=debug"
3848
SCCACHE_SERVER_PORT: "4225"
49+
SCCACHE_SERVER_LOG: "sccache=debug"
3950
permissions:
4051
id-token: write # This is required for configure-aws-credentials
4152
contents: read # This is required for actions/checkout
@@ -90,8 +101,9 @@ jobs:
90101
91102
set -x;
92103
93-
devcontainer-utils-install-sccache --repo rapidsai/sccache --version rapids;
94-
devcontainer-utils-start-sccache;
104+
sudo pkill -9 sccache || :;
105+
sudo rm -f .cache/*.log;
106+
devcontainer-utils-post-attach-command;
95107
96108
# Copy source folder into ~/stdexec
97109
cp -r "${GITHUB_WORKSPACE}"/stdexec ~/;
@@ -126,7 +138,7 @@ jobs:
126138
uses: actions/upload-artifact@v4
127139
with:
128140
name: sccache-client-logs-${{env.ARTIFACT_PREFIX}}-${{env.ARTIFACT_SUFFIX}}
129-
path: /home/coder/.cache/sccache*.log
141+
path: .cache/*.log
130142
compression-level: 9
131143

132144

0 commit comments

Comments
 (0)