Skip to content

Commit a91e533

Browse files
Merge pull request #4290 from AI-Hypercomputer:ci/test-duration-tracking
PiperOrigin-RevId: 953397876
2 parents 6e60e75 + 83666c2 commit a91e533

6 files changed

Lines changed: 361 additions & 0 deletions

File tree

.github/workflows/ci_pipeline.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,3 +416,15 @@ jobs:
416416
with:
417417
failed_run_id: '${{ github.run_id }}'
418418
secrets: inherit
419+
420+
track_performance:
421+
name: Track Test Performance
422+
needs: [tpu-tests, gpu-tests, cpu-tests]
423+
if: ${{ always() && (needs.cpu-tests.result == 'success' || needs.gpu-tests.result == 'success' || needs.tpu-tests.result == 'success') }}
424+
uses: ./.github/workflows/track_performance.yml
425+
permissions:
426+
contents: write
427+
id-token: write
428+
pull-requests: write
429+
secrets: inherit
430+

.github/workflows/run_tests_against_package.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ name: Run Tests Against MaxText Package
1919
on:
2020
workflow_call:
2121
inputs:
22+
flavor:
23+
required: true
24+
type: string
2225
device_type:
2326
required: true
2427
type: string
@@ -211,6 +214,7 @@ jobs:
211214
-v \
212215
-m "${FINAL_PYTEST_MARKER}" \
213216
--durations=0 \
217+
--junitxml=test-results-${INPUTS_FLAVOR}-${INPUTS_WORKER_GROUP}.xml \
214218
$PYTEST_COV_ARGS \
215219
$SPLIT_ARGS \
216220
${INPUTS_PYTEST_EXTRA_ARGS}
@@ -220,6 +224,7 @@ jobs:
220224
INPUTS_IS_SCHEDULED_RUN: ${{ inputs.is_scheduled_run }}
221225
INPUTS_PYTEST_MARKER: ${{ inputs.pytest_marker }}
222226
INPUTS_DEVICE_TYPE: ${{ inputs.device_type }}
227+
INPUTS_FLAVOR: ${{ inputs.flavor }}
223228
INPUTS_PYTEST_ADDOPTS: ${{ inputs.pytest_addopts }}
224229
INPUTS_TOTAL_WORKERS: ${{ inputs.total_workers }}
225230
INPUTS_WORKER_GROUP: ${{ inputs.worker_group }}
@@ -243,3 +248,11 @@ jobs:
243248
# If scheduled, upload to scheduled flag only. If PR, upload to regular flag only.
244249
flags: ${{ inputs.is_scheduled_run == 'true' && 'scheduled' || 'regular' }}
245250
verbose: true
251+
- name: Upload Test Results XML
252+
if: always()
253+
uses: actions/upload-artifact@v4
254+
with:
255+
name: test-results-${{ inputs.flavor }}-${{ inputs.worker_group }}-${{ github.run_id }}
256+
path: test-results-*.xml
257+
retention-days: 1
258+
if-no-files-found: ignore

.github/workflows/run_tests_coordinator.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ jobs:
109109

110110
uses: ./.github/workflows/run_tests_against_package.yml
111111
with:
112+
flavor: ${{ inputs.flavor }}
112113
# Infrastructure Mapping
113114
device_type: >-
114115
${{ fromJSON('{
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Copyright 2023-2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# This is a reusable workflow for tracking test performance, individual limits, and regressions.
16+
17+
name: Track Performance
18+
19+
on:
20+
workflow_call:
21+
22+
permissions:
23+
contents: write
24+
id-token: write
25+
pull-requests: write
26+
27+
jobs:
28+
track:
29+
name: Track Test Performance
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v5
33+
34+
- name: Mark git repositories as safe
35+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
36+
37+
- name: Download all test results
38+
uses: actions/download-artifact@v4
39+
with:
40+
path: test-results
41+
pattern: test-results-*-${{ github.run_id }}
42+
merge-multiple: true
43+
44+
- name: Fetch per-test baseline from gh-pages
45+
continue-on-error: true
46+
run: |
47+
git fetch origin gh-pages || true
48+
git show origin/gh-pages:dev/bench/per_test_baseline.json > per_test_baseline.json || echo "{}" > per_test_baseline.json
49+
50+
- name: Process Test Results (Limits, Regressions, and Dashboard Formatting)
51+
run: |
52+
python3 tests/utils/process_test_results.py test-results \
53+
--baseline per_test_baseline.json \
54+
--save-baseline new_per_test_baseline.json \
55+
--output-benchmark benchmark-results.json \
56+
${{ (github.event_name == 'schedule' && github.ref == 'refs/heads/main' || github.head_ref == 'ci/test-duration-tracking') && '--warn-only' || '' }}
57+
58+
- name: Push per-test baseline to gh-pages
59+
if: ${{ github.event_name == 'schedule' && github.ref == 'refs/heads/main' }}
60+
run: |
61+
git fetch origin gh-pages || true
62+
git worktree add gh-pages origin/gh-pages
63+
mkdir -p gh-pages/dev/bench
64+
cp new_per_test_baseline.json gh-pages/dev/bench/per_test_baseline.json
65+
cd gh-pages
66+
git config user.name "github-actions[bot]"
67+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
68+
git add dev/bench/per_test_baseline.json
69+
git diff-index --quiet HEAD || git commit -m "Update per-test baseline [skip ci]"
70+
git push origin HEAD:refs/heads/gh-pages
71+
cd ..
72+
git worktree remove gh-pages
73+
74+
- name: Track Test Durations (Macro-level Dashboard)
75+
uses: benchmark-action/github-action-benchmark@v1
76+
with:
77+
name: MaxText Test Execution Times
78+
tool: 'customSmallerIsBetter'
79+
output-file-path: benchmark-results.json
80+
github-token: ${{ secrets.GITHUB_TOKEN }}
81+
alert-threshold: '115%'
82+
comment-on-alert: true
83+
fail-on-alert: ${{ github.ref != 'refs/heads/main' && github.head_ref != 'ci/test-duration-tracking' }}
84+
auto-push: ${{ github.event_name == 'schedule' && github.ref == 'refs/heads/main' }}
85+
gh-pages-branch: 'gh-pages'
86+
benchmark-data-dir-path: 'dev/bench'

tests/conftest.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,3 +260,9 @@ def handle_gpu_only(request):
260260
has_gpu = False
261261
if not has_gpu:
262262
pytest.skip("Skipped: requires GPU hardware, none detected")
263+
264+
265+
def pytest_runtest_setup(item):
266+
"""Hook to inject markers as properties into the test item."""
267+
for marker in item.iter_markers():
268+
item.user_properties.append(("marker", marker.name))

0 commit comments

Comments
 (0)