Report E2E results to GitHub

e2e/BITRISE.md

@@ -66,6 +66,7 @@ The non-secret E2E defaults are defined in `e2e/bitrise.yml` under `app.envs`. D
 | `E2E_ANDROID_COMMAND_TIMEOUT_SECONDS` | `1800` | Per-command timeout for React Native Android artifact commands. |
 | `E2E_IOS_COMMAND_TIMEOUT_SECONDS` | `1800` | Per-command timeout for React Native iOS dependency, CocoaPods, archive, and export commands. |
 | `E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS` | `300` | Per-command timeout for package-suite matrix and suite commands. |
+| `E2E_REPORT_COMMAND_TIMEOUT_SECONDS` | `300` | Timeout for the GitHub reporting command. |
 | `E2E_RUBY_INSTALL_TIMEOUT_SECONDS` | `1800` | Timeout for installing the exact repository Ruby version from `.ruby-version`. |
 
 Secrets still need to be configured in Bitrise.io.
@@ -84,6 +85,16 @@ BrowserStack credentials are required by the `e2e-run-browserstack` workflow.
 | `BROWSERSTACK_USERNAME` | BrowserStack API username |
 | `BROWSERSTACK_ACCESS_KEY` | BrowserStack API access key |
 
+## GitHub reporting
+
+The report workflow uses a GitHub API token to create commit statuses, Check Runs, and sticky PR comments. Prefer the short-lived token generated by the Bitrise GitHub App over a long-lived personal access token.
+
+For Bitrise GitHub App projects, enable **Project settings > Repository > Extend GitHub App permissions to builds**. Bitrise then exposes the build-scoped GitHub App token as `GIT_HTTP_PASSWORD`; the report workflow maps it to `GITHUB_TOKEN` before running `e2e/scripts/report_e2e_results`.
+
+If the project is not using the Bitrise GitHub App, configure `GITHUB_TOKEN` as a Bitrise secret with equivalent permissions.
+
+Green runs update statuses and Check Runs without creating new PR comments. Failing runs update a sticky PR failure comment with direct BrowserStack evidence links.
+
 ## PR trigger
 
 Configure the Bitrise app to run the `e2e` pipeline for pull requests once the skeleton is ready to validate in Bitrise.

e2e/bitrise.yml

@@ -22,6 +22,7 @@ app:
     - E2E_ANDROID_COMMAND_TIMEOUT_SECONDS: "1800"
     - E2E_IOS_COMMAND_TIMEOUT_SECONDS: "1800"
     - E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS: "300"
+    - E2E_REPORT_COMMAND_TIMEOUT_SECONDS: "300"
     - E2E_RUBY_INSTALL_TIMEOUT_SECONDS: "1800"
 
 pipelines:
@@ -307,13 +308,27 @@ workflows:
   e2e-report:
     steps:
       - git-clone@8: {}
+      - restore-cache@3:
+          inputs:
+            - key: |-
+                e2e-ruby-linux-{{ checksum ".ruby-version" }}
       - pull-intermediate-files@1:
           inputs:
-            - artifact_sources: |-
-                e2e-run-browserstack
+            - artifact_sources: e2e-run-browserstack.*
       - script@1:
-          title: Placeholder E2E report
+          title: Report E2E results to GitHub
           inputs:
             - content: |-
                 set -euo pipefail
-                echo "Phase 2 placeholder for aggregate E2E reporting"
+                source e2e/scripts/bitrise_ci_helpers
+                e2e_log "Checking GitHub reporting configuration"
+                if [ -z "${GITHUB_TOKEN:-}" ]; then
+                  : "${GIT_HTTP_PASSWORD:?GITHUB_TOKEN or Bitrise GitHub App GIT_HTTP_PASSWORD is required. Enable Project settings > Repository > Extend GitHub App permissions to builds.}"
+                  export GITHUB_TOKEN="$GIT_HTTP_PASSWORD"
+                fi
+                : "${BITRISE_GIT_COMMIT:?BITRISE_GIT_COMMIT is required}"
+                : "${BITRISE_PULL_REQUEST:?BITRISE_PULL_REQUEST is required}"
+                e2e_prepare_ruby
+                results_root="${E2E_BROWSERSTACK_RESULTS_DIR:-$BITRISE_DEPLOY_DIR/e2e/results}"
+                e2e_log "Reporting E2E results to GitHub"
+                e2e_run_with_timeout "${E2E_REPORT_COMMAND_TIMEOUT_SECONDS:-300}" e2e/scripts/report_e2e_results --results-root "$results_root"

e2e/lib/e2e_github_reporter.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+class E2EGitHubReporter
+  COMMENT_MARKER = "<!-- checkout-kit-e2e-report -->"
+
+  def initialize(results, repository:, sha:, pr_number:, token: nil)
+    @results = results
+    @repository = repository
+    @sha = sha
+    @pr_number = pr_number
+    @token = token
+  end
+
+  def publish!
+    commit_status_payloads.each { |payload| post_json("/repos/#{@repository}/statuses/#{@sha}", payload) }
+    post_json("/repos/#{@repository}/check-runs", check_run_payload)
+    sync_failure_comment
+  end
+
+  def markdown_summary
+    lines = []
+    lines << "## Checkout Kit E2E results"
+    lines << ""
+    lines << "| Run | Status | Target | Platform | OS track | Device | Suite |"
+    lines << "|---|---|---|---|---|---|---|"
+    @results.each do |result|
+      lines << "| `#{result.fetch("id")}` | #{status_icon(result)} #{result.fetch("status")} | #{result.fetch("target")} | #{result.fetch("platform")} | #{result.fetch("os_track")} | #{result.fetch("device_selector")} → #{result.fetch("resolved_device")} | `#{result.fetch("execute")}` |"
+    end
+    failure_lines = failed_results.flat_map { |result| failure_details(result) }
+    unless failure_lines.empty?
+      lines << ""
+      lines << "## Failures"
+      lines << ""
+      lines << "> BrowserStack artifacts require BrowserStack access. Sign in to [BrowserStack App Automate](https://app-automate.browserstack.com/dashboard/v2/builds) before opening artifact links."
+      lines.concat(failure_lines)
+    end
+    lines.join("\n")
+  end
+
+  def commit_status_payloads
+    @results.map do |result|
+      {
+        state: result.fetch("passed") ? "success" : "failure",
+        context: result.fetch("status_context"),
+        description: status_description(result),
+        target_url: browserstack_build_url(result)
+      }
+    end
+  end
+
+  def failure_comment_body
+    return nil if failed_results.empty?
+
+    [COMMENT_MARKER, markdown_summary].join("\n")
+  end
+
+  private
+
+  def check_run_payload
+    conclusion = failed_results.empty? ? "success" : "failure"
+    {
+      name: "Checkout Kit E2E",
+      head_sha: @sha,
+      status: "completed",
+      conclusion: conclusion,
+      output: {
+        title: "Checkout Kit E2E #{conclusion}",
+        summary: markdown_summary
+      }
+    }
+  end
+
+  def sync_failure_comment
+    body = failure_comment_body
+    existing = existing_failure_comment
+    if body
+      if existing
+        patch_json("/repos/#{@repository}/issues/comments/#{existing.fetch("id")}", {body: body})
+      else
+        post_json("/repos/#{@repository}/issues/#{@pr_number}/comments", {body: body})
+      end
+    elsif existing
+      patch_json("/repos/#{@repository}/issues/comments/#{existing.fetch("id")}", {body: "#{COMMENT_MARKER}\n✅ Checkout Kit E2E failures resolved."})
+    end
+  end
+
+  def existing_failure_comment
+    get_json("/repos/#{@repository}/issues/#{@pr_number}/comments").find do |comment|
+      comment.fetch("body", "").include?(COMMENT_MARKER)
+    end
+  end
+
+  def failed_results
+    @results.reject { |result| result.fetch("passed") }
+  end
+
+  def failure_details(result)
+    lines = []
+    lines << ""
+    lines << "### `#{result.fetch("id")}`"
+    lines << ""
+    lines << "- Device selector: `#{result.fetch("device_selector")}`"
+    lines << "- Resolved device: `#{result.fetch("resolved_device")}`"
+    lines << "- Suite: `#{result.fetch("execute")}`"
+    lines << "- BrowserStack build: [Open BrowserStack build](#{browserstack_build_url(result)})"
+    result.fetch("failed_tests", []).each do |testcase|
+      lines << "- Failed test: `#{testcase.fetch("name", "unknown")}` (`#{testcase.fetch("status", "unknown")}`)"
+      link_fields.each do |label, key|
+        value = testcase[key]
+        lines << "  - #{label}: [Open #{link_text(label)}](#{value})" if value && !value.empty?
+      end
+    end
+    lines
+  end
+
+  def link_fields
+    {
+      "Video" => "video",
+      "Screenshot" => "screenshots",
+      "Maestro commands" => "maestro_commands",
+      "Maestro log" => "maestro_log",
+      "Device log" => "device_log",
+      "Network log" => "network_log"
+    }
+  end
+
+  def link_text(label)
+    label.start_with?("Maestro") ? label : label.downcase
+  end
+
+  def status_icon(result)
+    result.fetch("passed") ? "✅" : "❌"
+  end
+
+  def status_description(result)
+    description = result.fetch("passed") ? "passed" : "#{result.fetch("status")} on #{result.fetch("resolved_device")}" 
+    description[0, 140]
+  end
+
+  def browserstack_build_url(result)
+    "https://app-automate.browserstack.com/dashboard/v2/builds/#{result.fetch("build_id")}" 
+  end
+
+  def get_json(path)
+    execute_request(Net::HTTP::Get.new(path))
+  end
+
+  def post_json(path, body)
+    request = Net::HTTP::Post.new(path)
+    request.body = JSON.generate(body)
+    execute_json_request(request)
+  end
+
+  def patch_json(path, body)
+    request = Net::HTTP::Patch.new(path)
+    request.body = JSON.generate(body)
+    execute_json_request(request)
+  end
+
+  def execute_json_request(request)
+    request["Content-Type"] = "application/json"
+    execute_request(request)
+  end
+
+  def execute_request(request)
+    raise "GitHub token is required" unless @token
+
+    request["Accept"] = "application/vnd.github+json"
+    request["Authorization"] = "Bearer #{@token}"
+    response = Net::HTTP.start("api.github.com", 443, use_ssl: true) { |http| http.request(request) }
+    body = response.body.to_s.empty? ? {} : JSON.parse(response.body)
+    return body if response.is_a?(Net::HTTPSuccess)
+
+    raise "GitHub request failed #{response.code}: #{body}"
+  end
+end

e2e/scripts/report_e2e_results

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+require_relative "../lib/e2e_github_reporter"
+
+options = {
+  results_root: ENV.fetch("E2E_BROWSERSTACK_RESULTS_DIR", File.join(Dir.pwd, "e2e-results")),
+  repository: ENV.fetch("GITHUB_REPOSITORY", "Shopify/checkout-kit"),
+  sha: ENV.fetch("BITRISE_GIT_COMMIT"),
+  pr_number: Integer(ENV.fetch("BITRISE_PULL_REQUEST")),
+  token: ENV.fetch("GITHUB_TOKEN")
+}
+
+OptionParser.new do |opts|
+  opts.on("--results-root PATH") { |path| options[:results_root] = path }
+  opts.on("--repository REPOSITORY") { |repository| options[:repository] = repository }
+  opts.on("--sha SHA") { |sha| options[:sha] = sha }
+  opts.on("--pr NUMBER", Integer) { |number| options[:pr_number] = number }
+end.parse!
+
+result_paths = Dir.glob(File.join(options.fetch(:results_root), "**", "result.json"))
+if result_paths.empty?
+  warn "No E2E result files found under #{options.fetch(:results_root)}"
+  exit 1
+end
+
+results = result_paths.sort.map { |path| JSON.parse(File.read(path)) }
+reporter = E2EGitHubReporter.new(
+  results,
+  repository: options.fetch(:repository),
+  sha: options.fetch(:sha),
+  pr_number: options.fetch(:pr_number),
+  token: options.fetch(:token)
+)
+reporter.publish!
+puts reporter.markdown_summary

e2e/test/bitrise_pipeline_test.rb

@@ -314,6 +314,31 @@ def test_react_native_ios_workflow_does_not_use_local_native_sdk_overrides
     refute_includes script, "--local"
   end
 
+  def test_report_workflow_posts_github_results
+    config = load_bitrise_config
+    script = workflow_script("e2e-report")
+    ruby_cache = workflow_step(config, "e2e-report", "restore-cache", key_prefix: "e2e-ruby-linux-")
+    pull_step = workflow_step(config, "e2e-report", "pull-intermediate-files")
+
+    assert_equal "e2e-run-browserstack.*", pull_step.fetch("inputs").find { |input| input.key?("artifact_sources") }.fetch("artifact_sources")
+    assert_includes cache_input(ruby_cache, "key"), '{{ checksum ".ruby-version" }}'
+    assert_equal "300", app_env_values.fetch("E2E_REPORT_COMMAND_TIMEOUT_SECONDS")
+    assert_includes script, "source e2e/scripts/bitrise_ci_helpers"
+    assert_includes script, "e2e_prepare_ruby"
+    assert_includes script, 'e2e_log "Checking GitHub reporting configuration"'
+    assert_includes script, 'if [ -z "${GITHUB_TOKEN:-}" ]; then'
+    assert_includes script, 'GIT_HTTP_PASSWORD'
+    assert_includes script, 'export GITHUB_TOKEN="$GIT_HTTP_PASSWORD"'
+    assert_includes script, 'Extend GitHub App permissions to builds'
+    assert_includes script, 'e2e_log "Reporting E2E results to GitHub"'
+    assert_includes script, 'e2e_run_with_timeout "${E2E_REPORT_COMMAND_TIMEOUT_SECONDS:-300}" e2e/scripts/report_e2e_results --results-root "$results_root"'
+    assert_includes script, "GITHUB_TOKEN"
+    assert_includes script, "BITRISE_GIT_COMMIT"
+    assert_includes script, "BITRISE_PULL_REQUEST"
+    assert_includes script, "e2e/scripts/report_e2e_results"
+    refute_includes script, "Phase 2 placeholder for aggregate E2E reporting"
+  end
+
   def test_bitrise_setup_docs_exist
     assert File.exist?("e2e/BITRISE.md")
   end