Skip to content

Commit 388845e

Browse files
ko3n1gclaude
ko3n1g
and
claude
authored
ci(feat): use AWS ephemeral runners for external contributors (#1892)
* feat(ci): use nemo-ci AWS ephemeral runners for external contributors Signed-off-by: oliver könig <okoenig@nvidia.com> * fix(ci): switch Automodel from Azure ACR to AWS ECR Replace nemoci.azurecr.io with 766267172432.dkr.ecr.us-east-1.amazonaws.com. Remove all Azure CLI install, login, and ACR login steps from build-container and test-template actions. Drop environment: nemo-ci (Azure-backed) from all jobs. Route the CPU unit-test job from linux-amd64-cpu16 to the AWS runner selected by is-not-external-contributor. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Signed-off-by: oliver könig <okoenig@nvidia.com> --------- Signed-off-by: oliver könig <okoenig@nvidia.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent c7098b3 commit 388845e

4 files changed

Lines changed: 38 additions & 170 deletions

File tree

‎.github/actions/build-container/action.yml‎

Lines changed: 1 addition & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,9 @@
1515
name: Build container
1616

1717
inputs:
18-
azure-client-id:
19-
description: "Azure Client ID"
20-
required: true
21-
azure-tenant-id:
22-
description: "Azure Tenant ID"
23-
required: true
24-
azure-subscription-id:
25-
description: "Azure Subscription ID"
26-
required: true
2718
dockerfile-path:
2819
description: "Path to dockerfile to build"
2920
required: true
30-
has-azure-credentials:
31-
description: "Has Azure credentials"
32-
required: false
33-
default: "false"
3421
PAT:
3522
description: "GitHub Personal Access Token"
3623
required: true
@@ -40,7 +27,7 @@ inputs:
4027
type: string
4128

4229
env:
43-
container-registry: nemoci.azurecr.io
30+
container-registry: 766267172432.dkr.ecr.us-east-1.amazonaws.com
4431

4532
runs:
4633
using: "composite"
@@ -68,35 +55,6 @@ runs:
6855
with:
6956
ref: ${{ steps.sha.outputs.main }}
7057

71-
- name: Setup python
72-
uses: actions/setup-python@v6
73-
with:
74-
python-version: 3.12
75-
76-
- name: Install Azure CLI
77-
shell: bash
78-
run: |
79-
echo "::group::Install Azure CLI"
80-
# Create systemd override for proper dependencies
81-
for i in 1 2 3; do
82-
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash && break
83-
echo "Attempt $i failed, retrying in 10s..."
84-
sleep 10
85-
done
86-
echo "::endgroup::"
87-
88-
- name: Azure Login
89-
uses: azure/login@v2
90-
with:
91-
client-id: ${{ inputs.azure-client-id }}
92-
tenant-id: ${{ inputs.azure-tenant-id }}
93-
subscription-id: ${{ inputs.azure-subscription-id }}
94-
95-
- name: Azure ACR Login
96-
shell: bash
97-
run: |
98-
az acr login --name nemoci
99-
10058
- name: Install GH CLI
10159
shell: bash
10260
run: |

‎.github/actions/test-template/action.yml‎

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -34,19 +34,6 @@ inputs:
3434
description: "Run tests on CPU only"
3535
required: false
3636
default: "false"
37-
azure-client-id:
38-
description: "Azure Client ID"
39-
required: true
40-
azure-tenant-id:
41-
description: "Azure Tenant ID"
42-
required: true
43-
azure-subscription-id:
44-
description: "Azure Subscription ID"
45-
required: true
46-
has-azure-credentials:
47-
description: "Has Azure credentials"
48-
required: false
49-
default: "false"
5037
PAT:
5138
description: "GitHub Personal Access Token"
5239
required: true
@@ -63,31 +50,6 @@ inputs:
6350
runs:
6451
using: "composite"
6552
steps:
66-
- name: Install Azure CLI
67-
if: ${{ inputs.has-azure-credentials == 'true' }}
68-
shell: bash
69-
run: |
70-
echo "::group::Install Azure CLI"
71-
for i in 1 2 3; do
72-
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash && break
73-
echo "Attempt $i failed, retrying in 10s..."
74-
sleep 10
75-
done
76-
echo "::endgroup::"
77-
78-
- name: Azure Login
79-
if: ${{ inputs.has-azure-credentials == 'true' }}
80-
uses: azure/login@v2
81-
with:
82-
client-id: ${{ inputs.azure-client-id }}
83-
tenant-id: ${{ inputs.azure-tenant-id }}
84-
subscription-id: ${{ inputs.azure-subscription-id }}
85-
86-
- name: Azure ACR Login
87-
shell: bash
88-
run: |
89-
az acr login --name nemoci
90-
9153
- name: Install uuidgen
9254
shell: bash -x -e -u -o pipefail {0}
9355
if: ${{ contains(inputs.runner, 'aws') }}

‎.github/workflows/cicd-main.yml‎

Lines changed: 35 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,16 @@ permissions:
3131
pull-requests: read
3232

3333
env:
34-
container-registry: nemoci.azurecr.io
34+
container-registry: 766267172432.dkr.ecr.us-east-1.amazonaws.com
3535

3636
jobs:
3737
pre-flight:
3838
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.80.1
3939
with:
40-
default_runner_prefix: ${{ vars.DEFAULT_RUNNER_PREFIX }}
41-
non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_RUNNER_PREFIX }}
42-
default_test_data_path: ${{ vars.DEFAULT_TEST_DATA_PATH }}
43-
non_nvidia_test_data_path: ${{ vars.NON_NVIDIA_TEST_DATA_PATH }}
40+
default_runner_prefix: nemo-ci-aws-gpu-x2
41+
non_nvidia_runner_prefix: nemo-ci-aws-gpu-x2-ephemeral
42+
default_test_data_path: /mnt/datadrive/TestData/nemo-fw/TestData
43+
non_nvidia_test_data_path: /mnt/datadrive/TestData/nemo-fw/TestData
4444
sso_users_filename: ${{ vars.SSO_USERS_FILENAME }}
4545
secrets:
4646
NVIDIA_MANAGEMENT_ORG_PAT: ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }}
@@ -200,19 +200,14 @@ jobs:
200200
|| needs.pre-flight.outputs.force_run_all == 'true'
201201
)
202202
&& !cancelled()
203-
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
204-
environment: nemo-ci
203+
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
205204
steps:
206205
- name: Checkout
207206
uses: actions/checkout@v6
208207
- name: main
209208
uses: ./.github/actions/build-container
210209
with:
211-
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
212-
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
213-
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
214210
dockerfile-path: "./docker/Dockerfile"
215-
has-azure-credentials: "true"
216211
PAT: ${{ secrets.PAT }}
217212
repo-name: ${{ vars.CI_REPO_NAME || 'Automodel' }}
218213

@@ -223,16 +218,15 @@ jobs:
223218
matrix:
224219
include:
225220
- test-name: L0_Unit_Tests_CPU
226-
runner: linux-amd64-cpu16
221+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
227222
cpu-only: true
228223
- test-name: L0_Unit_Tests_GPU
229-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
224+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
230225
cpu-only: false
231226
timeout: 30
232227
needs: [pre-flight, cicd-wait-in-queue, cicd-container-build]
233228
runs-on: ${{ matrix.runner }}
234229
name: ${{ matrix.test-name }}
235-
environment: nemo-ci
236230
if: |
237231
(
238232
success()
@@ -249,10 +243,6 @@ jobs:
249243
timeout: ${{ matrix.timeout || 10 }}
250244
is-unit-test: "true"
251245
cpu-only: ${{ matrix.cpu-only || false }}
252-
has-azure-credentials: "true"
253-
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
254-
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
255-
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
256246
PAT: ${{ secrets.PAT }}
257247
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
258248
container-image: ${{ env.container-registry }}/${{ vars.CI_CONTAINER_NAME || 'automodel' }}:${{ github.sha }}
@@ -267,57 +257,56 @@ jobs:
267257
- test-name: L2_Pretrain_and_KD
268258
test-folder: llm_pretrain_and_kd
269259
timeout: 20
270-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
260+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
271261
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
272262
- test-name: L2_HF_DCP
273263
test-folder: hf_dcp
274264
timeout: 40
275-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
265+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
276266
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
277267
- test-name: L2_HF_PEFT
278268
test-folder: hf_peft
279269
timeout: 30
280-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
270+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
281271
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
282272
- test-name: L2_HF_Transformer
283273
test-folder: hf_transformer
284274
timeout: 20
285-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
275+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
286276
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
287277
- test-name: L2_HF_Transformer_Finetune
288278
test-folder: hf_transformer_finetune
289279
timeout: 40
290-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
280+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
291281
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
292282
- test-name: L2_HF_Transformer_LLM
293283
test-folder: hf_transformer_llm
294284
timeout: 40
295-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
285+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
296286
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
297287
- test-name: L2_HF_Transformer_VLM
298288
test-folder: hf_transformer_vlm
299289
timeout: 60
300-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
290+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
301291
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
302292
- test-name: L2_Datasets
303293
test-folder: data
304294
timeout: 20
305-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
295+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
306296
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
307297
- test-name: L2_Context_Parallel
308298
test-folder: context_parallel
309299
timeout: 20
310-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
300+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
311301
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
312302
- test-name: L2_Retrieval
313303
test-folder: retrieval
314304
timeout: 20
315-
runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2
305+
runner: ${{ needs.pre-flight.outputs.runner_prefix }}
316306
test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
317307
needs: [pre-flight, cicd-unit-tests]
318308
runs-on: ${{ matrix.runner }}
319309
name: ${{ matrix.test-name }}
320-
environment: nemo-ci
321310
if: |
322311
(
323312
success()
@@ -334,10 +323,6 @@ jobs:
334323
test-folder: ${{ matrix.test-folder }}
335324
timeout: ${{ matrix.timeout }}
336325
is_unit_test: "false"
337-
has-azure-credentials: "true"
338-
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
339-
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
340-
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
341326
PAT: ${{ secrets.PAT }}
342327
test-data-path: ${{ matrix.test-data-path }}
343328
container-image: ${{ env.container-registry }}/${{ vars.CI_CONTAINER_NAME || 'automodel' }}:${{ github.sha }}
@@ -393,7 +378,6 @@ jobs:
393378
&& needs.pre-flight.outputs.is_ci_workload == 'false'
394379
&& !cancelled()
395380
&& vars.ENABLE_CODECOV == 'true'
396-
environment: nemo-ci
397381
steps:
398382
- name: Generate fake coverage report
399383
uses: actions/github-script@v8
@@ -463,3 +447,20 @@ jobs:
463447
path: |
464448
.coverage
465449
include-hidden-files: true
450+
451+
cleanup-taint-node:
452+
runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
453+
needs:
454+
- pre-flight
455+
- Nemo_CICD_Test
456+
- Coverage
457+
- Coverage_Fake
458+
if: |
459+
always()
460+
&& !cancelled()
461+
&& needs.pre-flight.outputs.is_deployment_workflow != 'true'
462+
steps:
463+
- name: Taint node for cleanup
464+
if: contains(runner.name, 'ephemeral')
465+
shell: bash
466+
run: taint-node.sh

‎.github/workflows/config/.secrets.baseline‎

Lines changed: 2 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@
135135
"filename": ".github/actions/build-container/action.yml",
136136
"hashed_secret": "de4ecef3e7306203de51b2689836ae209612f4b9",
137137
"is_verified": false,
138-
"line_number": 156
138+
"line_number": 121
139139
}
140140
],
141141
".github/workflows/release-nightly-docs.yml": [
@@ -156,15 +156,6 @@
156156
"line_number": 62
157157
}
158158
],
159-
"examples/benchmark/configs/peft/nemotron_super_v3_lora.yaml": [
160-
{
161-
"type": "Base64 High Entropy String",
162-
"filename": "examples/benchmark/configs/peft/nemotron_super_v3_lora.yaml",
163-
"hashed_secret": "4702220fec9261c7d9b0c427aceeca118b66c702",
164-
"is_verified": false,
165-
"line_number": 63
166-
}
167-
],
168159
"examples/llm_finetune/nemotron/nemotron_super_v3_hellaswag.yaml": [
169160
{
170161
"type": "Base64 High Entropy String",
@@ -183,50 +174,6 @@
183174
"line_number": 42
184175
}
185176
],
186-
"examples/vlm_finetune/nemotron/parse-ft-tutorial/parse_finetune_tutorial.ipynb": [
187-
{
188-
"type": "Hex High Entropy String",
189-
"filename": "examples/vlm_finetune/nemotron/parse-ft-tutorial/parse_finetune_tutorial.ipynb",
190-
"hashed_secret": "710c91edd7984b025f47830ec2ba0506894eed3f",
191-
"is_verified": false,
192-
"line_number": 64
193-
},
194-
{
195-
"type": "Hex High Entropy String",
196-
"filename": "examples/vlm_finetune/nemotron/parse-ft-tutorial/parse_finetune_tutorial.ipynb",
197-
"hashed_secret": "e1f9a3f35c6bff77f847a39b4c4a5c9eb2892ecb",
198-
"is_verified": false,
199-
"line_number": 657
200-
},
201-
{
202-
"type": "Hex High Entropy String",
203-
"filename": "examples/vlm_finetune/nemotron/parse-ft-tutorial/parse_finetune_tutorial.ipynb",
204-
"hashed_secret": "4f248bc40ab0f68e2dceb22e38c52828ee276179",
205-
"is_verified": false,
206-
"line_number": 698
207-
},
208-
{
209-
"type": "Hex High Entropy String",
210-
"filename": "examples/vlm_finetune/nemotron/parse-ft-tutorial/parse_finetune_tutorial.ipynb",
211-
"hashed_secret": "d73541f2527eda15a3f3ab0018e4047afc81c7eb",
212-
"is_verified": false,
213-
"line_number": 752
214-
},
215-
{
216-
"type": "Hex High Entropy String",
217-
"filename": "examples/vlm_finetune/nemotron/parse-ft-tutorial/parse_finetune_tutorial.ipynb",
218-
"hashed_secret": "336e39a34addaaa1dcefb999629f7106e531126e",
219-
"is_verified": false,
220-
"line_number": 771
221-
},
222-
{
223-
"type": "Hex High Entropy String",
224-
"filename": "examples/vlm_finetune/nemotron/parse-ft-tutorial/parse_finetune_tutorial.ipynb",
225-
"hashed_secret": "b98ce2f8214f1c209f6371e4247784ef35088cfb",
226-
"is_verified": false,
227-
"line_number": 826
228-
}
229-
],
230177
"tests/unit_tests/config/test_allowed_import_prefixes.py": [
231178
{
232179
"type": "Secret Keyword",
@@ -276,5 +223,5 @@
276223
}
277224
]
278225
},
279-
"generated_at": "2026-04-03T21:56:52Z"
226+
"generated_at": "2026-04-18T10:18:13Z"
280227
}

0 commit comments

Comments
 (0)