ci: Extract test names for flaky test issues

.github/FLAKY_CI_FAILURE_TEMPLATE.md

@@ -1,5 +1,5 @@
 ---
-title: '[Flaky CI]: {{ env.JOB_NAME }}'
+title: '[Flaky CI]: {{ env.JOB_NAME }} - {{ env.TEST_NAME }}'
 labels: Tests
 ---
 
@@ -13,7 +13,7 @@ Other / Unknown
 
 ### Name of Test
 
-_Not available - check the run link for details_
+{{ env.TEST_NAME }}
 
 ### Link to Test Run
 

.github/workflows/build.yml

@@ -1125,6 +1125,7 @@ jobs:
     runs-on: ubuntu-24.04
     permissions:
       issues: write
+      checks: read
     steps:
       - name: Check out current commit
         if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure')
@@ -1171,35 +1172,59 @@ jobs:
               const jobName = job.name;
               const jobUrl = job.html_url;
 
-              // Replace template variables
-              const vars = {
-                'JOB_NAME': jobName,
-                'RUN_LINK': jobUrl
-              };
-
-              let title = frontmatter.match(/title:\s*'(.*)'/)[1];
-              let issueBody = bodyTemplate;
-              for (const [key, value] of Object.entries(vars)) {
-                const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g');
-                title = title.replace(pattern, value);
-                issueBody = issueBody.replace(pattern, value);
+              // Fetch annotations from the check run to extract failed test names
+              let testNames = [];
+              try {
+                const annotations = await github.paginate(github.rest.checks.listAnnotations, {
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  check_run_id: job.id,
+                  per_page: 100
+                });
+
+                const testAnnotations = annotations.filter(a => a.annotation_level === 'failure' && a.path !== '.github');
+                testNames = [...new Set(testAnnotations.map(a => a.title || a.path))];
+              } catch (e) {
+                console.log(`Could not fetch annotations for ${jobName}: ${e.message}`);
               }
 
-              const existingIssue = existing.find(i => i.title === title);
-
-              if (existingIssue) {
-                console.log(`Issue already exists for ${jobName}: #${existingIssue.number}`);
-                continue;
+              // If no test names found, fall back to one issue per job
+              if (testNames.length === 0) {
+                testNames = ['Unknown test'];
               }
 
-              const newIssue = await github.rest.issues.create({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                title: title,
-                body: issueBody.trim(),
-                labels: ['Tests']
-              });
-              console.log(`Created issue #${newIssue.data.number} for ${jobName}`);
+              // Create one issue per failing test for proper deduplication
+              for (const testName of testNames) {
+                const vars = {
+                  'JOB_NAME': jobName,
+                  'RUN_LINK': jobUrl,
+                  'TEST_NAME': testName
+                };
+
+                let title = frontmatter.match(/title:\s*'(.*)'/)[1];
+                let issueBody = bodyTemplate;
+                for (const [key, value] of Object.entries(vars)) {
+                  const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g');
+                  title = title.replace(pattern, value);
+                  issueBody = issueBody.replace(pattern, value);
+                }
+
+                const existingIssue = existing.find(i => i.title === title);
+
+                if (existingIssue) {
+                  console.log(`Issue already exists for "${testName}" in ${jobName}: #${existingIssue.number}`);
+                  continue;
+                }
+
+                const newIssue = await github.rest.issues.create({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  title: title,
+                  body: issueBody.trim(),
+                  labels: ['Tests']
+                });
+                console.log(`Created issue #${newIssue.data.number} for "${testName}" in ${jobName}`);
+              }
             }
 
       - name: Check for failures

dev-packages/browser-integration-tests/playwright.config.ts

@@ -30,7 +30,7 @@ const config: PlaywrightTestConfig = {
     },
   ],
 
-  reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+  reporter: process.env.CI ? [['list'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
 
   globalSetup: require.resolve('./playwright.setup.ts'),
   globalTeardown: require.resolve('./playwright.teardown.ts'),

dev-packages/test-utils/src/playwright-config.ts

@@ -37,7 +37,7 @@ export function getPlaywrightConfig(
     /* In dev mode some apps are flaky, so we allow retry there... */
     retries: testEnv === 'development' ? 3 : 0,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-    reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+    reporter: process.env.CI ? [['line'], ['github'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
       /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

packages/nuxt/vite.config.ts

@@ -3,6 +3,7 @@ import baseConfig from '../../vite/vite.config';
 export default {
   ...baseConfig,
   test: {
+    ...baseConfig.test,
     environment: 'jsdom',
     setupFiles: ['./test/vitest.setup.ts'],
     typecheck: {

vite/vite.config.ts

@@ -19,7 +19,9 @@ export default defineConfig({
         'vite.config.*',
       ],
     },
-    reporters: process.env.CI ? ['default', ['junit', { classnameTemplate: '{filepath}' }]] : ['default'],
+    reporters: process.env.CI
+      ? ['default', 'github-actions', ['junit', { classnameTemplate: '{filepath}' }]]
+      : ['default'],
     outputFile: {
       junit: 'vitest.junit.xml',
     },