Skip to content

Commit e6618df

Browse files
kevalmorabia97kevalmorabia97
committed
Move regression tests to separate workflow file and folder name
Signed-off-by: Keval Morabia <28916987+kevalmorabia97@users.noreply.github.com>
1 parent c34ca45 commit e6618df

7 files changed

Lines changed: 138 additions & 21 deletions

File tree

‎.github/workflows/bump_uv_lock.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ name: Bump uv.lock
33
on:
44
schedule:
55
- cron: "0 9 * * 1" # Every Monday at 9:00 UTC
6-
workflow_dispatch: # On-demand
6+
workflow_dispatch:
7+
# On-demand
78

89
permissions:
910
contents: write

‎.github/workflows/code_quality.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@ on:
55
branches: [main, release/*, feature/*]
66
schedule:
77
- cron: "0 0 * * *" # Nightly
8-
workflow_dispatch: # On-demand
8+
workflow_dispatch:
9+
# On-demand
10+
911

10-
# Cancel previous runs if new commit is pushed to the same PR
1112
concurrency:
13+
# Cancel previous runs if new commit is pushed to the same PR
1214
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
1315
cancel-in-progress: true
1416

‎.github/workflows/gpu_tests.yml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,6 @@ jobs:
4848
tests/gpu/**
4949
tests/gpu_megatron/**
5050
tests/gpu_trtllm/**
51-
tests/gpu_regression/**
52-
examples/speculative_decoding/**
53-
examples/dataset/**
54-
modelopt_recipes/general/speculative_decoding/**
55-
tools/launcher/**
5651
fail_on_initial_diff_error: true
5752
wait-checks:
5853
needs: [check-file-changes]
@@ -75,9 +70,6 @@ jobs:
7570
timeout: 60
7671
container_image: pytorch:26.01-py3
7772
# tests/gpu/_extensions/test_onnx_extensions.py fails for newer containers until https://github.com/tbenthompson/cppimport/pull/98
78-
- example: gpu_regression
79-
timeout: 15
80-
container_image: pytorch:26.01-py3
8173
- example: gpu_megatron
8274
timeout: 45
8375
container_image: nemo:26.04

‎.github/workflows/pages.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,18 @@ on:
88
branches: [main]
99
schedule:
1010
- cron: "0 0 * * *" # Nightly
11-
workflow_dispatch: # On-demand
11+
workflow_dispatch:
12+
# On-demand
13+
1214

13-
# Cancel previous runs if new commit is pushed
1415
concurrency:
16+
# Cancel previous runs if new commit is pushed
1517
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
1618
cancel-in-progress: true
1719

1820
permissions:
19-
contents: write # push to gh-pages branch
20-
pull-requests: write # post/update preview URL comment on PRs
21+
contents: write # push to gh-pages branch
22+
pull-requests: write # post/update preview URL comment on PRs
2123

2224
jobs:
2325
build-docs:

‎.github/workflows/regression_tests.yml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
name: Regression tests
2+
3+
on:
4+
push:
5+
branches: ["pull-request/[0-9]+"]
6+
# NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used
7+
schedule:
8+
- cron: "0 0 * * *" # Nightly
9+
workflow_dispatch:
10+
# On-demand
11+
12+
13+
concurrency:
14+
# Cancel previous runs if new commit is pushed to the same PR
15+
group: ${{ github.workflow }}-${{ startsWith(github.ref, 'refs/heads/pull-request/') && github.ref || github.sha }}
16+
cancel-in-progress: true
17+
18+
jobs:
19+
check-file-changes:
20+
if: startsWith(github.ref, 'refs/heads/pull-request/')
21+
runs-on: ubuntu-latest
22+
outputs:
23+
any_changed: ${{ steps.changed-tests.outputs.any_changed }}
24+
steps:
25+
- uses: actions/checkout@v6
26+
with:
27+
fetch-depth: 0
28+
- id: get-pr-info
29+
uses: nv-gha-runners/get-pr-info@main
30+
# Get commit from main branch that is present in the PR to use as base for changed files
31+
- id: calculate-merge-base
32+
env:
33+
PR_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }}
34+
BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}
35+
run: |
36+
(echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") | tee --append "${GITHUB_OUTPUT}"
37+
- name: Check for changes in test-relevant directories
38+
id: changed-tests
39+
uses: step-security/changed-files@v46.0.5
40+
with:
41+
base_sha: ${{ steps.calculate-merge-base.outputs.merge-base }}
42+
sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }}
43+
files: |
44+
.github/workflows/regression_tests.yml
45+
modelopt/torch/**
46+
noxfile.py
47+
pyproject.toml
48+
tests/regression/**
49+
examples/speculative_decoding/**
50+
examples/dataset/**
51+
modelopt_recipes/general/speculative_decoding/**
52+
tools/launcher/**
53+
fail_on_initial_diff_error: true
54+
wait-checks:
55+
needs: [check-file-changes]
56+
if: needs.check-file-changes.outputs.any_changed == 'true'
57+
uses: ./.github/workflows/_wait_for_checks.yml
58+
permissions:
59+
checks: read
60+
secrets: inherit
61+
with:
62+
match_pattern: "^DCO$|^linux$" # Wait for DCO and Unit tests / linux to pass
63+
delay: 300s
64+
regression-tests-pr:
65+
needs: [check-file-changes, wait-checks]
66+
if: needs.check-file-changes.outputs.any_changed == 'true'
67+
strategy: &regression_strategy
68+
fail-fast: false
69+
matrix:
70+
include:
71+
- example: regression
72+
timeout: 15
73+
container_image: pytorch:26.01-py3
74+
runs-on: linux-amd64-gpu-rtxpro6000-latest-1
75+
timeout-minutes: ${{ matrix.timeout }}
76+
container: &regression_container
77+
image: nvcr.io/nvidia/${{ matrix.container_image }}
78+
env:
79+
GIT_DEPTH: 1000 # For correct version
80+
PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages
81+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
82+
steps: &regression_steps
83+
- uses: actions/checkout@v6
84+
- uses: nv-gha-runners/setup-proxy-cache@main
85+
- name: Setup environment variables
86+
run: |
87+
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV
88+
- name: Run regression tests
89+
env:
90+
COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml
91+
COVERAGE_FILE: ${{ github.workspace }}/.coverage
92+
run: |
93+
pip install nox
94+
nox -s ${{ matrix.example }}
95+
- name: Upload regression coverage to Codecov
96+
uses: codecov/codecov-action@v5
97+
with:
98+
token: ${{ secrets.CODECOV_TOKEN }}
99+
files: coverage.xml
100+
flags: regression
101+
fail_ci_if_error: false # test may be skipped if relevant file changes are not detected
102+
verbose: true
103+
regression-tests-non-pr:
104+
if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }}
105+
strategy: *regression_strategy
106+
runs-on: linux-amd64-gpu-rtxpro6000-latest-2
107+
timeout-minutes: ${{ matrix.timeout }}
108+
container: *regression_container
109+
steps: *regression_steps
110+
regression-pr-required-check:
111+
# Run even if regression-tests-pr is skipped
112+
if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }}
113+
needs: [check-file-changes, regression-tests-pr]
114+
runs-on: ubuntu-latest
115+
steps:
116+
- name: Required regression tests did not succeed
117+
if: |
118+
needs.check-file-changes.result != 'success' ||
119+
(needs.check-file-changes.outputs.any_changed == 'true' && needs.regression-tests-pr.result != 'success')
120+
run: exit 1

‎noxfile.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,18 +128,18 @@ def gpu_megatron(session):
128128
session.run("python", "-m", "pytest", "tests/gpu_megatron", *_cov_args())
129129

130130

131-
# Container: nvcr.io/nvidia/pytorch:26.01-py3 or later
131+
# Container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc10 or later
132132
@nox.session(venv_backend="none")
133-
def gpu_regression(session):
133+
def gpu_trtllm(session):
134134
session.run("python", "-m", "pip", "install", "-e", ".[hf,dev-test]")
135-
session.run("python", "-m", "pytest", "tests/gpu_regression", *_cov_args())
135+
session.run("python", "-m", "pytest", "tests/gpu_trtllm", *_cov_args())
136136

137137

138-
# Container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc10 or later
138+
# Container: nvcr.io/nvidia/pytorch:26.01-py3 or later
139139
@nox.session(venv_backend="none")
140-
def gpu_trtllm(session):
140+
def regression(session):
141141
session.run("python", "-m", "pip", "install", "-e", ".[hf,dev-test]")
142-
session.run("python", "-m", "pytest", "tests/gpu_trtllm", *_cov_args())
142+
session.run("python", "-m", "pytest", "tests/regression", *_cov_args())
143143

144144

145145
# ─── Code quality ─────────────────────────────────────────────────────────────

tests/gpu_regression/torch/speculative/test_dflash.py renamed to tests/regression/torch/speculative/test_dflash.py

File renamed without changes.

0 commit comments

Comments
 (0)