ci: Extract test names for flaky test issues

.github/FLAKY_CI_FAILURE_TEMPLATE.md

@@ -1,5 +1,5 @@
 ---
-title: '[Flaky CI]: {{ env.JOB_NAME }}'
+title: '[Flaky CI]: {{ env.JOB_NAME }} - {{ env.TEST_NAME }}'
 labels: Tests
 ---
 
@@ -13,7 +13,7 @@ Other / Unknown
 
 ### Name of Test
 
-_Not available - check the run link for details_
+{{ env.TEST_NAME }}
 
 ### Link to Test Run
 

.github/workflows/build.yml

@@ -1155,82 +1155,25 @@ jobs:
     runs-on: ubuntu-24.04
     permissions:
       issues: write
+      checks: read
     steps:
       - name: Check out current commit
-        if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure')
+        if: contains(needs.*.result, 'failure')
         uses: actions/checkout@v6
         with:
-          sparse-checkout: .github
+          sparse-checkout: |
+            .github
+            scripts
 
       - name: Create issues for failed jobs
-        if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure')
+        if: contains(needs.*.result, 'failure')
         uses: actions/github-script@v7
         with:
           script: |
-            const fs = require('fs');
-
-            // Fetch actual job details from the API to get descriptive names
-            const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              run_id: context.runId,
-              per_page: 100
-            });
-
-            const failedJobs = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)'));
-
-            if (failedJobs.length === 0) {
-              console.log('No failed jobs found');
-              return;
-            }
-
-            // Read and parse template
-            const template = fs.readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8');
-            const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
-
-            // Get existing open issues with Tests label
-            const existing = await github.paginate(github.rest.issues.listForRepo, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              state: 'open',
-              labels: 'Tests',
-              per_page: 100
-            });
-
-            for (const job of failedJobs) {
-              const jobName = job.name;
-              const jobUrl = job.html_url;
-
-              // Replace template variables
-              const vars = {
-                'JOB_NAME': jobName,
-                'RUN_LINK': jobUrl
-              };
-
-              let title = frontmatter.match(/title:\s*'(.*)'/)[1];
-              let issueBody = bodyTemplate;
-              for (const [key, value] of Object.entries(vars)) {
-                const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g');
-                title = title.replace(pattern, value);
-                issueBody = issueBody.replace(pattern, value);
-              }
-
-              const existingIssue = existing.find(i => i.title === title);
-
-              if (existingIssue) {
-                console.log(`Issue already exists for ${jobName}: #${existingIssue.number}`);
-                continue;
-              }
-
-              const newIssue = await github.rest.issues.create({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                title: title,
-                body: issueBody.trim(),
-                labels: ['Tests']
-              });
-              console.log(`Created issue #${newIssue.data.number} for ${jobName}`);
-            }
+            const { default: run } = await import(
+              `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs`
+            );
+            await run({ github, context, core });
 
       - name: Check for failures
         if: cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')

dev-packages/browser-integration-tests/playwright.config.ts

@@ -30,7 +30,7 @@ const config: PlaywrightTestConfig = {
     },
   ],
 
-  reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+  reporter: process.env.CI ? [['list'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
 
   globalSetup: require.resolve('./playwright.setup.ts'),
   globalTeardown: require.resolve('./playwright.teardown.ts'),

dev-packages/test-utils/src/playwright-config.ts

@@ -37,7 +37,7 @@ export function getPlaywrightConfig(
     /* In dev mode some apps are flaky, so we allow retry there... */
     retries: testEnv === 'development' ? 3 : 0,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-    reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+    reporter: process.env.CI ? [['line'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
       /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

packages/core/test/lib/session.test.ts

@@ -4,6 +4,11 @@
 import { timestampInSeconds } from '../../src/utils/time';
 
 describe('Session', () => {
+  // TEMPORARY: Guaranteed failure to verify issue creation in CI
+  it('TEMPORARY - should fail to test annotation extraction', () => {
+    expect(true).toBe(false);
+  });
+
   it('initializes with the proper defaults', () => {
     const newSession = makeSession();
     const session = newSession.toJSON();

packages/nuxt/vite.config.ts

@@ -3,6 +3,7 @@ import baseConfig from '../../vite/vite.config';
 export default {
   ...baseConfig,
   test: {
+    ...baseConfig.test,
     environment: 'jsdom',
     setupFiles: ['./test/vitest.setup.ts'],
     typecheck: {

scripts/report-ci-failures.mjs

@@ -0,0 +1,109 @@
+/**
+ * CI Failure Reporter script.
+ *
+ * Creates GitHub issues for tests that fail on the develop branch.
+ * For each failed job in the workflow run, it fetches check run annotations
+ * to identify individual failing tests, then creates one issue per failing
+ * test using the FLAKY_CI_FAILURE_TEMPLATE.md template. Existing open issues
+ * with matching titles are skipped to avoid duplicates.
+ *
+ * Intended to be called from a GitHub Actions workflow via actions/github-script:
+ *
+ *   const { default: run } = await import(
+ *     `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs`
+ *   );
+ *   await run({ github, context, core });
+ */
+
+import { readFileSync } from 'node:fs';
+
+export default async function run({ github, context, core }) {
+  const { owner, repo } = context.repo;
+
+  // Fetch actual job details from the API to get descriptive names
+  const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
+    owner,
+    repo,
+    run_id: context.runId,
+    per_page: 100,
+  });
+
+  const failedJobs = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)'));
+
+  if (failedJobs.length === 0) {
+    core.info('No failed jobs found');
+    return;
+  }
+
+  // Read and parse template
+  const template = readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8');
+  const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+
+  // Get existing open issues with Tests label
+  const existing = await github.paginate(github.rest.issues.listForRepo, {
+    owner,
+    repo,
+    state: 'open',
+    labels: 'Tests',
+    per_page: 100,
+  });
+
+  for (const job of failedJobs) {
+    const jobName = job.name;
+    const jobUrl = job.html_url;
+
+    // Fetch annotations from the check run to extract failed test names
+    let testNames = [];
+    try {
+      const annotations = await github.paginate(github.rest.checks.listAnnotations, {
+        owner,
+        repo,
+        check_run_id: job.id,
+        per_page: 100,
+      });
+
+      const testAnnotations = annotations.filter(a => a.annotation_level === 'failure' && a.path !== '.github');
+      testNames = [...new Set(testAnnotations.map(a => a.title || a.path))];
+    } catch (e) {
+      core.info(`Could not fetch annotations for ${jobName}: ${e.message}`);
+    }
+
+    // If no test names found, fall back to one issue per job
+    if (testNames.length === 0) {
+      testNames = ['Unknown test'];
+    }
+
+    // Create one issue per failing test for proper deduplication
+    for (const testName of testNames) {
+      const vars = {
+        JOB_NAME: jobName,
+        RUN_LINK: jobUrl,
+        TEST_NAME: testName,
+      };
+
+      let title = frontmatter.match(/title:\s*'(.*)'/)[1];
+      let issueBody = bodyTemplate;
+      for (const [key, value] of Object.entries(vars)) {
+        const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g');
+        title = title.replace(pattern, value);
+        issueBody = issueBody.replace(pattern, value);
+      }
+
+      const existingIssue = existing.find(i => i.title === title);
+
+      if (existingIssue) {
+        core.info(`Issue already exists for "${testName}" in ${jobName}: #${existingIssue.number}`);
+        continue;
+      }
+
+      const newIssue = await github.rest.issues.create({
+        owner,
+        repo,
+        title,
+        body: issueBody.trim(),
+        labels: ['Tests'],
+      });
+      core.info(`Created issue #${newIssue.data.number} for "${testName}" in ${jobName}`);
+    }
+  }
+}

vite/vite.config.ts

@@ -19,7 +19,9 @@ export default defineConfig({
         'vite.config.*',
       ],
     },
-    reporters: process.env.CI ? ['default', ['junit', { classnameTemplate: '{filepath}' }]] : ['default'],
+    reporters: process.env.CI
+      ? ['default', 'github-actions', ['junit', { classnameTemplate: '{filepath}' }]]
+      : ['default'],
     outputFile: {
       junit: 'vitest.junit.xml',
     },