From 7d6f887af4a42f5fc89d9e0d8228d4655a377954 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Mon, 29 Jun 2026 16:52:53 +0100 Subject: [PATCH] Report E2E results to GitHub Assisted-By: devx/6c1e3ad5-96c8-4972-b087-da7ff7b195c3 --- e2e/BITRISE.md | 10 ++ e2e/bitrise.yml | 24 +++- e2e/lib/e2e_github_reporter.rb | 200 +++++++++++++++++++++++++++++++++ e2e/scripts/report_e2e_results | 38 +++++++ 4 files changed, 268 insertions(+), 4 deletions(-) create mode 100644 e2e/lib/e2e_github_reporter.rb create mode 100755 e2e/scripts/report_e2e_results diff --git a/e2e/BITRISE.md b/e2e/BITRISE.md index 39310350..88f65d28 100644 --- a/e2e/BITRISE.md +++ b/e2e/BITRISE.md @@ -83,6 +83,16 @@ BrowserStack credentials are required by the `e2e-run-browserstack` workflow. | `BROWSERSTACK_USERNAME` | BrowserStack API username | | `BROWSERSTACK_ACCESS_KEY` | BrowserStack API access key | +## GitHub reporting + +The report workflow uses a GitHub API token to create commit statuses, Check Runs, and sticky PR comments. Prefer the short-lived token generated by the Bitrise GitHub App over a long-lived personal access token. + +For Bitrise GitHub App projects, enable **Project settings > Repository > Extend GitHub App permissions to builds**. Bitrise then exposes the build-scoped GitHub App token as `GIT_HTTP_PASSWORD`; the report workflow maps it to `GITHUB_TOKEN` before running `e2e/scripts/report_e2e_results`. + +If the project is not using the Bitrise GitHub App, configure `GITHUB_TOKEN` as a Bitrise secret with equivalent permissions. + +Green runs update statuses and Check Runs without creating new PR comments. Failing runs update a sticky PR failure comment with direct BrowserStack evidence links. + ## PR trigger Configure the Bitrise app to run the `e2e` pipeline for pull requests once the skeleton is ready to validate in Bitrise. diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 03b489e3..3283a10c 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -358,13 +358,29 @@ workflows: e2e-report: steps: - git-clone@8: {} + - restore-cache@3: + inputs: + - key: |- + e2e-ruby-linux-{{ checksum ".ruby-version" }} - pull-intermediate-files@1: inputs: - - artifact_sources: |- - e2e-run-browserstack + - artifact_sources: e2e-run-browserstack.* + - bundle::setup-ruby: {} - script@1: - title: Placeholder E2E report + title: Report E2E results to GitHub + timeout: 900 + no_output_timeout: 900 inputs: - content: |- set -euo pipefail - echo "Phase 2 placeholder for aggregate E2E reporting" + source e2e/scripts/bitrise_ci_helpers + e2e_log "Checking GitHub reporting configuration" + if [ -z "${GITHUB_TOKEN:-}" ]; then + : "${GIT_HTTP_PASSWORD:?GITHUB_TOKEN or Bitrise GitHub App GIT_HTTP_PASSWORD is required. Enable Project settings > Repository > Extend GitHub App permissions to builds.}" + export GITHUB_TOKEN="$GIT_HTTP_PASSWORD" + fi + : "${BITRISE_GIT_COMMIT:?BITRISE_GIT_COMMIT is required}" + : "${BITRISE_PULL_REQUEST:?BITRISE_PULL_REQUEST is required}" + results_root="${E2E_BROWSERSTACK_RESULTS_DIR:-$BITRISE_DEPLOY_DIR/e2e/results}" + e2e_log "Reporting E2E results to GitHub" + e2e/scripts/report_e2e_results --results-root "$results_root" diff --git a/e2e/lib/e2e_github_reporter.rb b/e2e/lib/e2e_github_reporter.rb new file mode 100644 index 00000000..79e5b289 --- /dev/null +++ b/e2e/lib/e2e_github_reporter.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +require "json" +require "net/http" +require "uri" + +class E2EGitHubReporter + COMMENT_MARKER = "" + + def initialize(results, repository:, sha:, pr_number:, token: nil) + @results = results + @repository = repository + @sha = sha + @pr_number = pr_number + @token = token + end + + def publish! + commit_status_payloads.each { |payload| post_json("/repos/#{@repository}/statuses/#{@sha}", payload) } + post_json("/repos/#{@repository}/check-runs", check_run_payload) + sync_failure_comment + end + + def markdown_summary + lines = [] + lines << "## Checkout Kit E2E results" + lines << "" + lines << "| Status | Suite | Target | Platform | OS track | Device |" + lines << "|---|---|---|---|---|---|" + @results.each do |result| + lines << "| #{status_icon(result)} | `#{result.fetch("execute")}` | #{result.fetch("target")} | #{result.fetch("platform")} | #{result.fetch("os_track")} | #{device_cell(result)} |" + end + failure_lines = failed_results.flat_map { |result| failure_details(result) } + unless failure_lines.empty? + lines << "" + lines << "## Failures" + lines << "" + lines << "> BrowserStack artifacts require BrowserStack access. Sign in to [BrowserStack App Automate](https://app-automate.browserstack.com/dashboard/v2/builds) before opening artifact links." + lines.concat(failure_lines) + end + lines.join("\n") + end + + def commit_status_payloads + @results.map do |result| + { + state: result.fetch("passed") ? "success" : "failure", + context: result.fetch("status_context"), + description: status_description(result), + target_url: browserstack_build_url(result) + } + end + end + + def failure_comment_body + return nil if failed_results.empty? + + [COMMENT_MARKER, markdown_summary].join("\n") + end + + private + + def check_run_payload + conclusion = failed_results.empty? ? "success" : "failure" + { + name: "Checkout Kit E2E", + head_sha: @sha, + status: "completed", + conclusion: conclusion, + output: { + title: "Checkout Kit E2E #{conclusion}", + summary: markdown_summary + } + } + end + + def sync_failure_comment + body = failure_comment_body + existing = existing_failure_comment + if body + if existing + patch_json("/repos/#{@repository}/issues/comments/#{existing.fetch("id")}", {body: body}) + else + post_json("/repos/#{@repository}/issues/#{@pr_number}/comments", {body: body}) + end + elsif existing + patch_json("/repos/#{@repository}/issues/comments/#{existing.fetch("id")}", {body: "#{COMMENT_MARKER}\n✅ Checkout Kit E2E failures resolved."}) + end + end + + def existing_failure_comment + get_json("/repos/#{@repository}/issues/#{@pr_number}/comments").find do |comment| + comment.fetch("body", "").include?(COMMENT_MARKER) + end + end + + def failed_results + @results.reject { |result| result.fetch("passed") } + end + + def failure_details(result) + lines = [] + lines << "" + lines << "### #{failure_heading(result)}" + lines << "" + lines << "| Test | Status | Artifacts |" + lines << "|---|---|---|" + tests = result.fetch("failed_tests", []) + if tests.empty? + lines << "| — | #{status_icon(result)} | #{artifact_links(nil, result)} |" + else + tests.each do |testcase| + lines << "| `#{testcase.fetch("name", "unknown")}` | ❌ | #{artifact_links(testcase, result)} |" + end + end + lines + end + + def failure_heading(result) + suite = File.basename(result.fetch("execute"), ".*") + "#{os_label(result.fetch("platform"))} — #{suite}" + end + + def artifact_links(testcase, result) + links = ["[BrowserStack](#{browserstack_build_url(result)})"] + if testcase + link_fields.each do |label, key| + value = testcase[key] + links << "[#{label}](#{value})" if value && !value.empty? + end + end + links.join(" · ") + end + + def link_fields + { + "Video" => "video", + "Screenshot" => "screenshots", + "Maestro commands" => "maestro_commands", + "Maestro log" => "maestro_log", + "Device log" => "device_log", + "Network log" => "network_log" + } + end + + def device_cell(result) + name, version = result.fetch("resolved_device").split("-", 2) + version ? "#{name}
#{os_label(result.fetch("platform"))} #{version}" : name + end + + def os_label(platform) + platform == "ios" ? "iOS" : "Android" + end + + def status_icon(result) + result.fetch("passed") ? "✅" : "❌" + end + + def status_description(result) + description = result.fetch("passed") ? "passed" : "#{result.fetch("status")} on #{result.fetch("resolved_device")}" + description[0, 140] + end + + def browserstack_build_url(result) + "https://app-automate.browserstack.com/dashboard/v2/builds/#{result.fetch("build_id")}" + end + + def get_json(path) + execute_request(Net::HTTP::Get.new(path)) + end + + def post_json(path, body) + request = Net::HTTP::Post.new(path) + request.body = JSON.generate(body) + execute_json_request(request) + end + + def patch_json(path, body) + request = Net::HTTP::Patch.new(path) + request.body = JSON.generate(body) + execute_json_request(request) + end + + def execute_json_request(request) + request["Content-Type"] = "application/json" + execute_request(request) + end + + def execute_request(request) + raise "GitHub token is required" unless @token + + request["Accept"] = "application/vnd.github+json" + request["Authorization"] = "Bearer #{@token}" + response = Net::HTTP.start("api.github.com", 443, use_ssl: true) { |http| http.request(request) } + body = response.body.to_s.empty? ? {} : JSON.parse(response.body) + return body if response.is_a?(Net::HTTPSuccess) + + raise "GitHub request failed #{response.code}: #{body}" + end +end diff --git a/e2e/scripts/report_e2e_results b/e2e/scripts/report_e2e_results new file mode 100755 index 00000000..17da8cfc --- /dev/null +++ b/e2e/scripts/report_e2e_results @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "optparse" +require_relative "../lib/e2e_github_reporter" + +options = { + results_root: ENV.fetch("E2E_BROWSERSTACK_RESULTS_DIR", File.join(Dir.pwd, "e2e-results")), + repository: ENV.fetch("GITHUB_REPOSITORY", "Shopify/checkout-kit"), + sha: ENV.fetch("BITRISE_GIT_COMMIT"), + pr_number: Integer(ENV.fetch("BITRISE_PULL_REQUEST")), + token: ENV.fetch("GITHUB_TOKEN") +} + +OptionParser.new do |opts| + opts.on("--results-root PATH") { |path| options[:results_root] = path } + opts.on("--repository REPOSITORY") { |repository| options[:repository] = repository } + opts.on("--sha SHA") { |sha| options[:sha] = sha } + opts.on("--pr NUMBER", Integer) { |number| options[:pr_number] = number } +end.parse! + +result_paths = Dir.glob(File.join(options.fetch(:results_root), "**", "result.json")) +if result_paths.empty? + warn "No E2E result files found under #{options.fetch(:results_root)}" + exit 1 +end + +results = result_paths.sort.map { |path| JSON.parse(File.read(path)) } +reporter = E2EGitHubReporter.new( + results, + repository: options.fetch(:repository), + sha: options.fetch(:sha), + pr_number: options.fetch(:pr_number), + token: options.fetch(:token) +) +reporter.publish! +puts reporter.markdown_summary