Skip to content

Commit 8e99636

Browse files
narendasannarendasan
committed
tests: Overhauling CI infra
ci(test-dx): grant id-token: write at ci.yml top level The reusable build jobs (build_linux/build_windows) request 'id-token: write' for OIDC; a reusable workflow can't exceed its caller's permission ceiling, so ci.yml's top-level 'contents: read' made GitHub reject it (startup_failure: "nested job 'build' is requesting 'id-token: write', but is only allowed 'id-token: none'"). Grant id-token: write + contents: read, matching the original per-platform entry workflows.
1 parent 72f4d36 commit 8e99636

26 files changed

Lines changed: 486 additions & 3785 deletions

‎.github/workflows/_linux-x86_64-core.yml‎

Lines changed: 0 additions & 671 deletions
This file was deleted.

‎.github/workflows/_test-linux.yml‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
name: _test-linux
2+
3+
# Manifest-driven build (+ optional test) for ONE Linux channel. Covers both:
4+
# * x86_64 — os=linux, architecture=x86_64, run-tests=true (build + suites)
5+
# * SBSA — os=linux-aarch64, architecture=aarch64, run-tests=false (build-only;
6+
# there are no aarch64 GPU test runners)
7+
#
8+
# One reusable serves both because the build (build_linux.yml) and test
9+
# (linux-test.yml) reusables are the same — only inputs differ. (GitHub requires
10+
# `uses:` to be a literal, so Windows, which needs build_windows/windows-test,
11+
# has its own _test-windows.yml.)
12+
13+
on:
14+
workflow_call:
15+
inputs:
16+
lane:
17+
description: "fast | full | nightly"
18+
required: true
19+
type: string
20+
use-rtx:
21+
description: "Test against TensorRT-RTX (x86_64 only)"
22+
type: boolean
23+
default: false
24+
name-prefix:
25+
description: "Display-name prefix (e.g. 'RTX - ', 'SBSA ')"
26+
type: string
27+
default: ""
28+
os:
29+
description: "generate_binary_build_matrix os: linux | linux-aarch64"
30+
type: string
31+
default: linux
32+
architecture:
33+
description: "x86_64 | aarch64"
34+
type: string
35+
default: x86_64
36+
run-tests:
37+
description: "Run the suite matrix after building (false = build-only, e.g. SBSA)"
38+
type: boolean
39+
default: true
40+
python-only:
41+
description: "Build the PYTHON_ONLY=1 wheel (no C++ runtime) via env_vars_python_only.txt"
42+
type: boolean
43+
default: false
44+
45+
jobs:
46+
generate-matrix:
47+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
48+
with:
49+
package-type: wheel
50+
os: ${{ inputs.os }}
51+
test-infra-repository: pytorch/test-infra
52+
test-infra-ref: main
53+
with-rocm: false
54+
with-cpu: false
55+
56+
filter-matrix:
57+
needs: [generate-matrix]
58+
outputs:
59+
matrix: ${{ steps.generate.outputs.matrix }}
60+
runs-on: ubuntu-latest
61+
steps:
62+
- uses: actions/setup-python@v6
63+
with:
64+
python-version: "3.11"
65+
- uses: actions/checkout@v6
66+
with:
67+
repository: pytorch/tensorrt
68+
- name: Generate matrix
69+
id: generate
70+
run: |
71+
set -eou pipefail
72+
MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }}
73+
LIMIT_PR=${{ inputs.lane == 'fast' && 'true' || 'false' }}
74+
MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")"
75+
echo "${MATRIX_BLOB}"
76+
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
77+
78+
build:
79+
needs: filter-matrix
80+
permissions:
81+
id-token: write
82+
contents: read
83+
strategy:
84+
fail-fast: false
85+
matrix:
86+
include:
87+
- repository: pytorch/tensorrt
88+
pre-script: packaging/pre_build_script.sh
89+
env-var-script: packaging/env_vars.txt
90+
post-script: packaging/post_build_script.sh
91+
smoke-test-script: packaging/smoke_test_script.sh
92+
package-name: torch_tensorrt
93+
display-name: ${{ inputs.name-prefix }}Build Linux ${{ inputs.architecture }} torch-tensorrt whl package
94+
name: ${{ matrix.display-name }}
95+
uses: ./.github/workflows/build_linux.yml
96+
with:
97+
repository: ${{ matrix.repository }}
98+
ref: ""
99+
test-infra-repository: pytorch/test-infra
100+
test-infra-ref: main
101+
build-matrix: ${{ needs.filter-matrix.outputs.matrix }}
102+
pre-script: ${{ matrix.pre-script }}
103+
# python-only swaps in the env_vars_python_only.txt (sets PYTHON_ONLY=1).
104+
env-var-script: ${{ inputs.python-only && 'packaging/env_vars_python_only.txt' || matrix.env-var-script }}
105+
post-script: ${{ matrix.post-script }}
106+
package-name: ${{ matrix.package-name }}
107+
smoke-test-script: ${{ matrix.smoke-test-script }}
108+
trigger-event: ${{ github.event_name }}
109+
architecture: ${{ inputs.architecture }}
110+
use-rtx: ${{ inputs.use-rtx }}
111+
pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple"
112+
113+
# Suite list from the manifest. Skipped for build-only channels (SBSA).
114+
suite-matrix:
115+
if: ${{ inputs.run-tests }}
116+
runs-on: ubuntu-latest
117+
outputs:
118+
matrix: ${{ steps.gen.outputs.matrix }}
119+
steps:
120+
- uses: actions/checkout@v6
121+
- uses: actions/setup-python@v6
122+
with:
123+
python-version: "3.11"
124+
- id: gen
125+
run: |
126+
set -euo pipefail
127+
variant=${{ inputs.use-rtx && 'rtx' || 'standard' }}
128+
json="$(python -m tests.ci matrix --platform linux-x86_64 --lane '${{ inputs.lane }}' --variant "${variant}")"
129+
echo "matrix=${json}" >> "$GITHUB_OUTPUT"
130+
echo "Lane '${{ inputs.lane }}' (${variant}) suites:"; echo "${json}" | python -m json.tool
131+
132+
# One test job per suite (auto-skips when suite-matrix is skipped, i.e. SBSA).
133+
test:
134+
needs: [filter-matrix, build, suite-matrix]
135+
strategy:
136+
fail-fast: false
137+
matrix: ${{ fromJson(needs.suite-matrix.outputs.matrix) }}
138+
uses: ./.github/workflows/linux-test.yml
139+
with:
140+
job-name: ${{ matrix.suite }}-${{ matrix.variant }}
141+
repository: "pytorch/tensorrt"
142+
ref: ""
143+
test-infra-repository: pytorch/test-infra
144+
test-infra-ref: main
145+
build-matrix: ${{ needs.filter-matrix.outputs.matrix }}
146+
pre-script: packaging/pre_build_script.sh
147+
use-rtx: ${{ inputs.use-rtx }}
148+
fail-on-empty: true
149+
script: |
150+
set -euo pipefail
151+
python -m tests.ci run "${{ matrix.suite }}" --variant "${{ matrix.variant }}"

.github/workflows/build-test-windows_rtx-python-only.yml renamed to .github/workflows/_test-windows.yml

Lines changed: 52 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,26 @@
1-
name: RTX - Python-only build and test Windows wheels
1+
name: _test-windows
22

3-
on:
4-
pull_request:
5-
push:
6-
branches:
7-
- main
8-
- nightly
9-
- release/*
10-
tags:
11-
# NOTE: Binary build pipelines should only get triggered on release candidate builds
12-
# Release candidate tags look like: v1.11.0-rc1
13-
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
14-
workflow_dispatch:
3+
# Manifest-driven build + test for the Windows channel. Mirrors _test-linux.yml
4+
# but uses build_windows.yml + windows-test.yml, and wraps the suite run in
5+
# vc_env_helper.bat (the MSVC env). Suite SELECTION is identical to Linux — the
6+
# manifest is platform-agnostic (`ci matrix --platform windows`); only the build
7+
# reusable and the execution wrapper differ.
158

16-
permissions:
17-
contents: read
9+
on:
10+
workflow_call:
11+
inputs:
12+
lane:
13+
description: "fast | full | nightly"
14+
required: true
15+
type: string
16+
use-rtx:
17+
description: "Test against TensorRT-RTX"
18+
type: boolean
19+
default: false
20+
name-prefix:
21+
description: "Display-name prefix (e.g. 'RTX - ')"
22+
type: string
23+
default: ""
1824

1925
jobs:
2026
generate-matrix:
@@ -44,10 +50,12 @@ jobs:
4450
run: |
4551
set -eou pipefail
4652
MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }}
47-
MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")"
53+
LIMIT_PR=${{ inputs.lane == 'fast' && 'true' || 'false' }}
54+
MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")"
4855
echo "${MATRIX_BLOB}"
4956
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
5057
58+
# Swap the build runner label for a GPU test runner (same as the legacy windows flow).
5159
substitute-runner:
5260
needs: filter-matrix
5361
outputs:
@@ -57,7 +65,7 @@ jobs:
5765
- name: Substitute runner
5866
id: substitute
5967
run: |
60-
echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT}
68+
echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> "${GITHUB_OUTPUT}"
6169
6270
build:
6371
needs: substitute-runner
@@ -73,7 +81,7 @@ jobs:
7381
env-script: packaging/vc_env_helper.bat
7482
smoke-test-script: packaging/smoke_test_windows.py
7583
package-name: torch_tensorrt
76-
display-name: RTX - Python-only build Windows torch-tensorrt-rtx whl package
84+
display-name: ${{ inputs.name-prefix }}Build Windows torch-tensorrt whl package
7785
name: ${{ matrix.display-name }}
7886
uses: ./.github/workflows/build_windows.yml
7987
with:
@@ -84,41 +92,45 @@ jobs:
8492
build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
8593
pre-script: ${{ matrix.pre-script }}
8694
env-script: ${{ matrix.env-script }}
87-
wheel-build-params: "--py-only --use-rtx"
8895
smoke-test-script: ${{ matrix.smoke-test-script }}
8996
package-name: ${{ matrix.package-name }}
9097
trigger-event: ${{ github.event_name }}
9198
timeout: 120
92-
use-rtx: true
9399

94-
dynamo-runtime-tests:
95-
name: ${{ matrix.display-name }}
96-
needs: [substitute-runner, build]
97-
if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }}
100+
suite-matrix:
101+
runs-on: ubuntu-latest
102+
outputs:
103+
matrix: ${{ steps.gen.outputs.matrix }}
104+
steps:
105+
- uses: actions/checkout@v6
106+
- uses: actions/setup-python@v6
107+
with:
108+
python-version: "3.11"
109+
- id: gen
110+
run: |
111+
set -euo pipefail
112+
variant=${{ inputs.use-rtx && 'rtx' || 'standard' }}
113+
json="$(python -m tests.ci matrix --platform windows --lane '${{ inputs.lane }}' --variant "${variant}")"
114+
echo "matrix=${json}" >> "$GITHUB_OUTPUT"
115+
echo "Lane '${{ inputs.lane }}' (${variant}) windows suites:"; echo "${json}" | python -m json.tool
116+
117+
test:
118+
needs: [substitute-runner, build, suite-matrix]
98119
strategy:
99120
fail-fast: false
100-
matrix:
101-
include:
102-
- repository: pytorch/tensorrt
103-
package-name: torch_tensorrt
104-
display-name: RTX - Python-only dynamo runtime tests
121+
matrix: ${{ fromJson(needs.suite-matrix.outputs.matrix) }}
105122
uses: ./.github/workflows/windows-test.yml
106123
with:
107-
job-name: rtx-python-only-dynamo-runtime-tests
108-
repository: ${{ matrix.repository }}
124+
job-name: ${{ matrix.suite }}-${{ matrix.variant }}
125+
repository: "pytorch/tensorrt"
109126
ref: ""
110127
test-infra-repository: pytorch/test-infra
111128
test-infra-ref: main
112129
build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
113130
pre-script: packaging/driver_upgrade.bat
114-
use-rtx: true
131+
use-rtx: ${{ inputs.use-rtx }}
132+
# vc_env_helper.bat sets the MSVC env, then runs the manifest runner; the
133+
# inner `python -m pytest` it spawns inherits that env.
115134
script: |
116135
set -euo pipefail
117-
pushd .
118-
cd tests/py/dynamo/runtime
119-
../../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml .
120-
popd
121-
122-
concurrency:
123-
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-python-only-${{ github.event_name == 'workflow_dispatch' }}
124-
cancel-in-progress: true
136+
packaging/vc_env_helper.bat python -m tests.ci run "${{ matrix.suite }}" --variant "${{ matrix.variant }}"

‎.github/workflows/blossom-ci.yml‎

Lines changed: 0 additions & 104 deletions
This file was deleted.

0 commit comments

Comments
 (0)