|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | +# |
| 4 | +# Nightly E2E Scorecard |
| 5 | +# |
| 6 | +# Aggregates overnight nightly-e2e workflow results into a scorecard |
| 7 | +# published to $GITHUB_STEP_SUMMARY. Identifies flaky jobs, computes |
| 8 | +# pass/fail/cancel breakdowns, and compares trends against the prior day. |
| 9 | +# |
| 10 | +# Runs daily at 12:00 UTC (8:00 AM ET), after the nightly suite completes. |
| 11 | +# Also supports manual dispatch for testing. |
| 12 | +# |
| 13 | +# Phase 3 adds an optional Slack webhook: if SLACK_WEBHOOK_URL is set in |
| 14 | +# repository secrets, the scorecard is POSTed to Slack. If absent, the |
| 15 | +# step succeeds silently. |
| 16 | + |
| 17 | +name: nightly-scorecard |
| 18 | + |
| 19 | +on: |
| 20 | + schedule: |
| 21 | + - cron: "0 12 * * *" |
| 22 | + workflow_dispatch: |
| 23 | + |
| 24 | +permissions: |
| 25 | + contents: read |
| 26 | + actions: read |
| 27 | + |
| 28 | +jobs: |
| 29 | + scorecard: |
| 30 | + if: github.repository == 'NVIDIA/NemoClaw' |
| 31 | + runs-on: ubuntu-latest |
| 32 | + steps: |
| 33 | + - name: Generate nightly scorecard |
| 34 | + id: scorecard |
| 35 | + uses: actions/github-script@v7 |
| 36 | + with: |
| 37 | + script: | |
| 38 | + // ── Config ────────────────────────────────────────────── |
| 39 | + const WORKFLOW_FILE = 'nightly-e2e.yaml'; |
| 40 | + const EXCLUDED_JOBS = new Set(['gpu-e2e', 'notify-on-failure']); |
| 41 | +
|
| 42 | + // ── Helpers ───────────────────────────────────────────── |
| 43 | + function dateRange(hoursAgo) { |
| 44 | + const now = new Date(); |
| 45 | + const since = new Date(now.getTime() - hoursAgo * 60 * 60 * 1000); |
| 46 | + return since.toISOString(); |
| 47 | + } |
| 48 | +
|
| 49 | + function formatDate(date) { |
| 50 | + return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); |
| 51 | + } |
| 52 | +
|
| 53 | + async function fetchRuns(since) { |
| 54 | + const runs = []; |
| 55 | + let page = 1; |
| 56 | + while (true) { |
| 57 | + const { data } = await github.rest.actions.listWorkflowRuns({ |
| 58 | + owner: context.repo.owner, |
| 59 | + repo: context.repo.repo, |
| 60 | + workflow_id: WORKFLOW_FILE, |
| 61 | + created: `>=${since}`, |
| 62 | + per_page: 100, |
| 63 | + page, |
| 64 | + }); |
| 65 | + runs.push(...data.workflow_runs); |
| 66 | + if (data.workflow_runs.length < 100) break; |
| 67 | + page++; |
| 68 | + } |
| 69 | + // Only include completed runs |
| 70 | + return runs.filter(r => r.status === 'completed'); |
| 71 | + } |
| 72 | +
|
| 73 | + async function fetchJobs(runId) { |
| 74 | + const jobs = []; |
| 75 | + let page = 1; |
| 76 | + while (true) { |
| 77 | + const { data } = await github.rest.actions.listJobsForWorkflowRun({ |
| 78 | + owner: context.repo.owner, |
| 79 | + repo: context.repo.repo, |
| 80 | + run_id: runId, |
| 81 | + per_page: 100, |
| 82 | + page, |
| 83 | + }); |
| 84 | + jobs.push(...data.jobs); |
| 85 | + if (data.jobs.length < 100) break; |
| 86 | + page++; |
| 87 | + } |
| 88 | + return jobs.filter(j => !EXCLUDED_JOBS.has(j.name)); |
| 89 | + } |
| 90 | +
|
| 91 | + function analyzeRuns(runs, jobDetails) { |
| 92 | + let success = 0; |
| 93 | + let failure = 0; |
| 94 | + let cancelled = 0; |
| 95 | + const jobFailures = {}; |
| 96 | +
|
| 97 | + for (const run of runs) { |
| 98 | + if (run.conclusion === 'success') { |
| 99 | + success++; |
| 100 | + } else if (run.conclusion === 'failure') { |
| 101 | + failure++; |
| 102 | + } else if (run.conclusion === 'cancelled') { |
| 103 | + cancelled++; |
| 104 | + } |
| 105 | +
|
| 106 | + const jobs = jobDetails.get(run.id) || []; |
| 107 | + for (const job of jobs) { |
| 108 | + if (job.conclusion === 'failure') { |
| 109 | + jobFailures[job.name] = (jobFailures[job.name] || 0) + 1; |
| 110 | + } |
| 111 | + } |
| 112 | + } |
| 113 | +
|
| 114 | + // Jobs failing 2+ times, sorted by frequency descending |
| 115 | + const flaky = Object.entries(jobFailures) |
| 116 | + .filter(([, count]) => count >= 2) |
| 117 | + .sort((a, b) => b[1] - a[1]); |
| 118 | +
|
| 119 | + return { total: runs.length, success, failure, cancelled, flaky }; |
| 120 | + } |
| 121 | +
|
| 122 | + // ── Phase 1: Fetch today's runs (last 24h) ───────────── |
| 123 | + const todaySince = dateRange(24); |
| 124 | + const todayRuns = await fetchRuns(todaySince); |
| 125 | +
|
| 126 | + const today = formatDate(new Date()); |
| 127 | +
|
| 128 | + if (todayRuns.length === 0) { |
| 129 | + const noRunsCard = [ |
| 130 | + `## 🌅 NemoClaw Nightly Scorecard — ${today}`, |
| 131 | + '', |
| 132 | + 'No nightly E2E runs completed in the last 24 hours.', |
| 133 | + '', |
| 134 | + `🔗 [Nightly E2E workflow](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/${WORKFLOW_FILE})`, |
| 135 | + ].join('\n'); |
| 136 | + core.summary.addRaw(noRunsCard); |
| 137 | + await core.summary.write(); |
| 138 | + core.setOutput('scorecard', noRunsCard); |
| 139 | + return; |
| 140 | + } |
| 141 | +
|
| 142 | + // Fetch job details for today's runs |
| 143 | + const todayJobDetails = new Map(); |
| 144 | + for (const run of todayRuns) { |
| 145 | + todayJobDetails.set(run.id, await fetchJobs(run.id)); |
| 146 | + } |
| 147 | + const todayStats = analyzeRuns(todayRuns, todayJobDetails); |
| 148 | +
|
| 149 | + // ── Phase 2: Fetch yesterday's runs (24–48h ago) ──────── |
| 150 | + const yesterdaySince = dateRange(48); |
| 151 | + const allRuns48h = await fetchRuns(yesterdaySince); |
| 152 | + // Filter to only runs from the 24–48h window (exclude today's) |
| 153 | + const todayRunIds = new Set(todayRuns.map(r => r.id)); |
| 154 | + const yesterdayRuns = allRuns48h.filter(r => !todayRunIds.has(r.id)); |
| 155 | +
|
| 156 | + let trendLine = ''; |
| 157 | + if (yesterdayRuns.length > 0) { |
| 158 | + const yesterdayJobDetails = new Map(); |
| 159 | + for (const run of yesterdayRuns) { |
| 160 | + yesterdayJobDetails.set(run.id, await fetchJobs(run.id)); |
| 161 | + } |
| 162 | + const yesterdayStats = analyzeRuns(yesterdayRuns, yesterdayJobDetails); |
| 163 | +
|
| 164 | + if (todayStats.success > yesterdayStats.success) { |
| 165 | + trendLine = `Trend: ↗️ Improving (yesterday: ${yesterdayStats.success} perfect → today: ${todayStats.success} perfect)`; |
| 166 | + } else if (todayStats.success < yesterdayStats.success) { |
| 167 | + trendLine = `Trend: ↘️ Degrading (yesterday: ${yesterdayStats.success} perfect → today: ${todayStats.success} perfect)`; |
| 168 | + } else { |
| 169 | + trendLine = `Trend: ➡️ Stable (${todayStats.success} perfect both days)`; |
| 170 | + } |
| 171 | + } else { |
| 172 | + trendLine = 'Trend: ⊘ No prior-day data for comparison'; |
| 173 | + } |
| 174 | +
|
| 175 | + // ── Build scorecard ───────────────────────────────────── |
| 176 | + // Determine the total jobs per run from the first run's job list |
| 177 | + const firstRunJobs = todayJobDetails.get(todayRuns[0].id) || []; |
| 178 | + const jobCount = firstRunJobs.length; |
| 179 | +
|
| 180 | + const lines = [ |
| 181 | + `## 🌅 NemoClaw Nightly Scorecard — ${today}`, |
| 182 | + '', |
| 183 | + `**Overnight runs:** ${todayStats.total} completed`, |
| 184 | + ` ✅ ${todayStats.success} perfect` + (jobCount > 0 ? ` (${jobCount}/${jobCount})` : ''), |
| 185 | + ` ❌ ${todayStats.failure} failures`, |
| 186 | + ` ⊘ ${todayStats.cancelled} cancelled`, |
| 187 | + ]; |
| 188 | +
|
| 189 | + if (todayStats.flaky.length > 0) { |
| 190 | + lines.push(''); |
| 191 | + lines.push('**Top flaky jobs** (failed 2+ times):'); |
| 192 | + const maxNameLen = Math.max(...todayStats.flaky.map(([name]) => name.length)); |
| 193 | + for (const [name, count] of todayStats.flaky) { |
| 194 | + const padded = name.padEnd(maxNameLen); |
| 195 | + lines.push(` \`${padded}\` — ${count} failure${count === 1 ? '' : 's'}`); |
| 196 | + } |
| 197 | + } |
| 198 | +
|
| 199 | + lines.push(''); |
| 200 | + lines.push(trendLine); |
| 201 | + lines.push(''); |
| 202 | + lines.push(`🔗 [Nightly E2E workflow](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/${WORKFLOW_FILE})`); |
| 203 | +
|
| 204 | + const scorecard = lines.join('\n'); |
| 205 | + core.summary.addRaw(scorecard); |
| 206 | + await core.summary.write(); |
| 207 | + core.setOutput('scorecard', scorecard); |
| 208 | +
|
| 209 | + # ── Phase 3: Optional Slack notification ─────────────────── |
| 210 | + - name: Post scorecard to Slack |
| 211 | + if: ${{ steps.scorecard.outputs.scorecard != '' }} |
| 212 | + uses: actions/github-script@v7 |
| 213 | + env: |
| 214 | + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} |
| 215 | + SCORECARD_TEXT: ${{ steps.scorecard.outputs.scorecard }} |
| 216 | + with: |
| 217 | + script: | |
| 218 | + const webhookUrl = process.env.SLACK_WEBHOOK_URL; |
| 219 | + if (!webhookUrl) { |
| 220 | + core.info('SLACK_WEBHOOK_URL not configured — skipping Slack notification'); |
| 221 | + return; |
| 222 | + } |
| 223 | +
|
| 224 | + const scorecard = process.env.SCORECARD_TEXT; |
| 225 | +
|
| 226 | + // Strip markdown formatting for Slack plain-text rendering |
| 227 | + const slackText = scorecard |
| 228 | + .replace(/^## /gm, '') |
| 229 | + .replace(/\*\*/g, '*') |
| 230 | + .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<$2|$1>'); |
| 231 | +
|
| 232 | + const resp = await fetch(webhookUrl, { |
| 233 | + method: 'POST', |
| 234 | + headers: { 'Content-Type': 'application/json' }, |
| 235 | + body: JSON.stringify({ text: slackText }), |
| 236 | + }); |
| 237 | +
|
| 238 | + if (!resp.ok) { |
| 239 | + core.warning(`Slack webhook returned ${resp.status}: ${await resp.text()}`); |
| 240 | + } else { |
| 241 | + core.info('Scorecard posted to Slack'); |
| 242 | + } |
0 commit comments