ci(actions): integration tests

.github/scripts/test_report_comment.dart

@@ -0,0 +1,225 @@
+// Generates a markdown PR comment summarising test results.
+//
+// Usage: dart .github/scripts/test_report_comment.dart <reports-dir> <output-file>
+//
+// Reads every *.json file in <reports-dir> (dart/flutter test
+// `--file-reporter=json` output, one JSON event per line) and writes a
+// markdown comment to <output-file>:
+//
+//   * a [!CAUTION] callout when at least one test failed, [!NOTE] otherwise
+//   * a per-package summary table (test count and success rate)
+//   * for packages with failures only: a collapsible per-file breakdown
+//     listing each test in files that contain at least one failing test
+
+import 'dart:convert';
+import 'dart:io';
+
+void main(List<String> args) {
+  if (args.length != 2) {
+    stderr.writeln(
+      'Usage: dart test_report_comment.dart <reports-dir> <output-file>',
+    );
+    exit(64);
+  }
+
+  final reportFiles = Directory(args.first)
+      .listSync()
+      .whereType<File>()
+      .where((f) => f.path.endsWith('.json'))
+      .toList()
+    ..sort((a, b) => a.path.compareTo(b.path));
+
+  final packages = [
+    for (final file in reportFiles) _PackageReport.parse(file),
+  ];
+
+  File(args.last).writeAsStringSync(_render(packages));
+}
+
+class _TestResult {
+  const _TestResult(this.name, this.result, this.skipped);
+
+  final String name;
+  final String result; // success | failure | error
+  final bool skipped;
+
+  bool get failed => result != 'success' && !skipped;
+}
+
+class _PackageReport {
+  _PackageReport(this.name);
+
+  /// Package name, derived from the report file name
+  /// (e.g. `clerk_auth.json` -> `clerk_auth`).
+  final String name;
+
+  /// Test results keyed by display path of the test file.
+  final byFile = <String, List<_TestResult>>{};
+
+  int get total => byFile.values.fold(0, (sum, tests) => sum + tests.length);
+
+  int get failed => byFile.values
+      .fold(0, (sum, tests) => sum + tests.where((t) => t.failed).length);
+
+  int get successRate => total == 0 ? 100 : ((total - failed) * 100) ~/ total;
+
+  static _PackageReport parse(File file) {
+    final fileName = file.uri.pathSegments.last;
+    final report =
+        _PackageReport(fileName.substring(0, fileName.length - '.json'.length));
+
+    final suitePaths = <int, String>{};
+    final groupNames = <int, String>{};
+    final testsById = <int, Map<String, dynamic>>{};
+
+    for (final line in file.readAsLinesSync()) {
+      if (line.trim().isEmpty) continue;
+
+      final Map<String, dynamic> event;
+      try {
+        event = json.decode(line) as Map<String, dynamic>;
+      } on FormatException {
+        continue; // tolerate non-JSON noise in the report file
+      }
+
+      switch (event['type']) {
+        case 'suite':
+          final suite = event['suite'] as Map<String, dynamic>;
+          suitePaths[suite['id'] as int] =
+              _displayPath(suite['path'] as String? ?? '');
+
+        case 'group':
+          final group = event['group'] as Map<String, dynamic>;
+          groupNames[group['id'] as int] = group['name'] as String? ?? '';
+
+        case 'testStart':
+          final test = event['test'] as Map<String, dynamic>;
+          testsById[test['id'] as int] = test;
+
+        case 'testDone':
+          // Hidden tests are bookkeeping entries (e.g. suite loading).
+          if (event['hidden'] == true) continue;
+          final test = testsById[event['testID']];
+          if (test == null) continue;
+          final path = suitePaths[test['suiteID']] ?? '(unknown file)';
+          report.byFile.putIfAbsent(path, () => []).add(
+                _TestResult(
+                  _displayName(test, groupNames),
+                  event['result'] as String? ?? 'error',
+                  event['skipped'] == true,
+                ),
+              );
+      }
+    }
+
+    return report;
+  }
+
+  /// Path of a test file relative to its package root.
+  static String _displayPath(String path) {
+    for (final marker in ['integration_test/', 'test/']) {
+      if (path.startsWith(marker)) return path;
+      final index = path.indexOf('/$marker');
+      if (index >= 0) return path.substring(index + 1);
+    }
+    return path.split('/').last;
+  }
+
+  /// Test name with its group prefix separated by ` · `.
+  ///
+  /// Group names and test names are cumulative in the JSON reporter output
+  /// (each includes its ancestors' names), so successive prefixes are
+  /// stripped to recover the bare name of each level.
+  static String _displayName(
+    Map<String, dynamic> test,
+    Map<int, String> groupNames,
+  ) {
+    final segments = <String>[];
+    var cumulative = '';
+
+    for (final id in (test['groupIDs'] as List?)?.cast<int>() ?? const []) {
+      final fullName = groupNames[id] ?? '';
+      if (fullName.isEmpty) continue;
+      var segment = fullName;
+      if (cumulative.isNotEmpty && fullName.startsWith(cumulative)) {
+        segment = fullName.substring(cumulative.length).trim();
+      }
+      if (segment.isNotEmpty) segments.add(segment);
+      cumulative = fullName;
+    }
+
+    var bareName = test['name'] as String? ?? '';
+    if (cumulative.isNotEmpty && bareName.startsWith(cumulative)) {
+      bareName = bareName.substring(cumulative.length).trim();
+    }
+
+    return [...segments, if (bareName.isNotEmpty) bareName].join(' · ');
+  }
+}
+
+String _render(List<_PackageReport> packages) {
+  final anyFailures = packages.any((p) => p.failed > 0);
+  final lines = <String>[
+    if (anyFailures) ...[
+      '[!CAUTION]',
+      'At least one test failed. Please review the test results below.',
+    ] else ...[
+      '[!NOTE]',
+      'All tests passed.',
+    ],
+    '',
+    '<details>',
+    '<summary><b>View Test Results</b></summary>',
+    '',
+    '| Package | Number of Tests | Success Rate |',
+    '| :--- | :--- | :--- |',
+    for (final package in packages)
+      '| ${_cell(package.name)} | ${package.total} | ${package.successRate}% |',
+    for (final package in packages.where((p) => p.failed > 0))
+      ..._renderPackageBreakdown(package),
+    '',
+    '</details>',
+  ];
+
+  return lines.map((line) => line.isEmpty ? '>' : '> $line').join('\n');
+}
+
+Iterable<String> _renderPackageBreakdown(_PackageReport package) sync* {
+  yield '';
+  yield '<details>';
+  yield '<summary>📂 <b>${_cell(package.name)}</b></summary>';
+
+  // Only files containing at least one failing test are listed, so the
+  // breakdown stays skimmable.
+  final failingFiles = package.byFile.entries
+      .where((entry) => entry.value.any((test) => test.failed))
+      .toList()
+    ..sort((a, b) => a.key.compareTo(b.key));
+
+  for (final entry in failingFiles) {
+    yield '';
+    yield '### ${entry.key}';
+    yield '';
+    yield '| Test | Result |';
+    yield '| :--- | :--- |';
+    for (final test in entry.value) {
+      final name = test.failed ? _redText(test.name) : _cell(test.name);
+      final icon = test.failed ? '❌' : (test.skipped ? '⏭️' : '✅');
+      yield '| $name | $icon |';
+    }
+  }
+
+  yield '';
+  yield '</details>';
+}
+
+/// Escapes a value for use inside a markdown table cell.
+String _cell(String value) => value.replaceAll('|', r'\|');
+
+/// Renders a failing test name in red via GitHub's LaTeX support.
+String _redText(String value) {
+  final escaped = value
+      .replaceAll(RegExp(r'[\\~^|]'), ' ')
+      .replaceAllMapped(RegExp(r'[#$%&_{}]'), (m) => '\\${m[0]}');
+  return '\$\\color{red}{\\text{$escaped}}\$';
+}

.github/workflows/main.yaml

@@ -6,6 +6,11 @@ on:
   pull_request:
     branches: [ "main" ]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  # Cancel superseded runs on PR branches, but let runs on main complete.
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   pr_check:
     name: PR Check Build
@@ -68,10 +73,101 @@ jobs:
         run: flutter test --file-reporter=json:../../reports/clerk_flutter.json
         working-directory: packages/clerk_flutter
 
+      - name: Upload test reports
+        uses: actions/upload-artifact@v4
+        if: always()                  # upload reports even if tests failed
+        with:
+          name: reports-unit
+          path: reports/*.json
+
+  integration_test:
+    name: Integration Tests (iOS Simulator)
+    runs-on: macos-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Checkout repo (self)
+        uses: actions/checkout@v4
+
+      - name: Install flutter
+        uses: flutter-actions/setup-flutter@v4
+        with:
+          version: '3.41.9'
+
+      - name: 'Install tools: melos and fvm'
+        run: dart pub global activate melos && dart pub global activate fvm
+
+      - name: Install FVM
+        run: fvm install
+
+      - name: Install top-level dependencies
+        run: dart pub get
+
+      - name: Run melos
+        run: melos bs && melos run get
+
+      - name: Boot iPhone simulator
+        id: simulator
+        run: |
+          UDID=$(xcrun simctl list devices available --json \
+            | jq -r '[.devices | to_entries[] | select(.key | contains("iOS")) | .value[] | select(.name | startswith("iPhone"))] | last | .udid')
+          if [ -z "$UDID" ] || [ "$UDID" = "null" ]; then
+            echo "No available iPhone simulator found" >&2
+            xcrun simctl list devices available
+            exit 1
+          fi
+          echo "udid=$UDID" >> "$GITHUB_OUTPUT"
+          xcrun simctl bootstatus "$UDID" -b
+
+      - name: Run integration tests in clerk_flutter_example
+        run: melos run test:integration -- -d ${{ steps.simulator.outputs.udid }} --file-reporter=json:$GITHUB_WORKSPACE/reports/clerk_flutter_integration.json
+
+      - name: Upload test reports
+        uses: actions/upload-artifact@v4
+        if: always()                  # upload reports even if tests failed
+        with:
+          name: reports-integration
+          path: reports/*.json
+
+  test_report:
+    name: Test Report
+    runs-on: ubuntu-latest
+    needs: [ pr_check, integration_test ]
+    if: ${{ !cancelled() }}           # report even if a test job failed
+    permissions:
+      contents: read
+      checks: write                   # dorny/test-reporter creates a check run
+      pull-requests: write            # sticky comment on the PR
+
+    steps:
+      - name: Checkout repo (self)
+        uses: actions/checkout@v4
+
+      - name: Download test reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: reports-*
+          merge-multiple: true
+          path: reports
+
       - name: Test Report
         uses: dorny/test-reporter@v1
-        if: success() || failure()    # run this step even if previous step failed
         with:
           name: 'Test Report'         # Name of the check run which will be created
           path: reports/*.json        # Path to test results
           reporter: flutter-json      # Format of test results
+
+      - name: Install dart
+        uses: dart-lang/setup-dart@v1
+        if: ${{ !cancelled() }}       # comment even if tests failed
+
+      - name: Generate test summary comment
+        if: ${{ !cancelled() }}
+        run: dart .github/scripts/test_report_comment.dart reports test_summary_comment.md
+
+      - name: Post test summary comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        if: ${{ !cancelled() && github.event_name == 'pull_request' }}
+        with:
+          header: test-results
+          path: test_summary_comment.md