From 9c5a564fd1ee2bad630d2f5821021085af145419 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Mon, 29 Jun 2026 16:55:20 +0100 Subject: [PATCH 1/2] Soft-launch E2E PR checks Assisted-By: devx/6c1e3ad5-96c8-4972-b087-da7ff7b195c3 --- e2e/BITRISE.md | 8 +++- e2e/RUNBOOK.md | 69 ++++++++++++++++++++++++++++++++ e2e/bitrise.yml | 6 ++- e2e/scripts/browserstack_maestro | 43 ++++++++++++++++---- e2e/test/soft_launch_test.rb | 46 +++++++++++++++++++++ 5 files changed, 163 insertions(+), 9 deletions(-) create mode 100644 e2e/RUNBOOK.md create mode 100644 e2e/test/soft_launch_test.rb diff --git a/e2e/BITRISE.md b/e2e/BITRISE.md index 9b946b93..3a24bc29 100644 --- a/e2e/BITRISE.md +++ b/e2e/BITRISE.md @@ -85,6 +85,12 @@ BrowserStack credentials are required by the `e2e-run-browserstack` workflow. | `BROWSERSTACK_USERNAME` | BrowserStack API username | | `BROWSERSTACK_ACCESS_KEY` | BrowserStack API access key | +BrowserStack artifact links in GitHub reports require access to BrowserStack App Automate. Sign in to [BrowserStack App Automate](https://app-automate.browserstack.com/dashboard/v2/builds) before opening build, video, screenshot, or log links. + +## Duplicate PR build cancellation + +Use Bitrise native Rolling builds instead of a repo-owned cancellation script. In Bitrise, open **Project settings > Builds > Build strategy**, enable **Abort builds triggered by pull requests**, and enable **Abort running builds** so duplicate in-progress PR pipelines are cancelled when a newer build starts. + ## GitHub reporting The report workflow uses a GitHub API token to create commit statuses, Check Runs, and sticky PR comments. Prefer the short-lived token generated by the Bitrise GitHub App over a long-lived personal access token. @@ -97,7 +103,7 @@ Green runs update statuses and Check Runs without creating new PR comments. Fail ## PR trigger -Configure the Bitrise app to run the `e2e` pipeline for pull requests once the skeleton is ready to validate in Bitrise. +`bitrise.yml` maps pull requests to the `e2e` pipeline with `trigger_map`. Keep the GitHub checks non-required while `E2E_STRICT=false`. ## Code signing diff --git a/e2e/RUNBOOK.md b/e2e/RUNBOOK.md new file mode 100644 index 00000000..d1406132 --- /dev/null +++ b/e2e/RUNBOOK.md @@ -0,0 +1,69 @@ +# Checkout Kit E2E Runbook + +## Rollout mode + +The Bitrise E2E pipeline starts in soft-fail mode: + +```bash +E2E_STRICT=false +``` + +In soft-fail mode, BrowserStack Maestro assertion failures are reported through GitHub commit statuses, Check Runs, and the sticky PR failure comment, but the BrowserStack workflow exits successfully so the initial rollout does not block PR progress. + +Set `E2E_STRICT=true` only after the suite has stabilized and the corresponding GitHub contexts are ready to become required checks. + +## Retry behavior + +BrowserStack API calls retry transient infrastructure responses once by default: + +```bash +E2E_BROWSERSTACK_API_RETRIES=1 +``` + +Retry applies to HTTP 429 and 5xx responses. Maestro assertion failures are not auto-retried by default so first-failure evidence is preserved. + +## Timeouts + +BrowserStack polling uses these defaults: + +```bash +E2E_BROWSERSTACK_TIMEOUT_SECONDS=1800 +E2E_BROWSERSTACK_POLL_SECONDS=30 +``` + +If polling times out, the runner attempts to stop the BrowserStack build before failing or soft-failing according to `E2E_STRICT`. + +## Local rerun notes + +Use the matrix row from a failure report to identify the app target, platform, OS track, and suite: + +```bash +ruby e2e/scripts/e2e_matrix expand --index +``` + +Use the reported resolved device to pin a rerun: + +```bash +E2E_DEVICE_OVERRIDE="" \ +E2E_STRICT=true \ +e2e/scripts/browserstack_maestro --index --suite-zip --output-dir +``` + +The app artifact environment variable for the matrix row must point at the `.apk` or `.ipa` artifact before rerunning. + +## Failure triage + +Use the GitHub Check Run or sticky PR failure comment first. Failure summaries should include Markdown links to: + +- BrowserStack build +- failed testcase +- video +- screenshot +- Maestro command log +- Maestro log +- device log +- network log when enabled + +BrowserStack artifact links require BrowserStack App Automate access. Sign in to [BrowserStack App Automate](https://app-automate.browserstack.com/dashboard/v2/builds) before opening evidence links. + +Avoid posting additional PR comments for green runs. diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index b2793d77..7365dcd2 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -25,6 +25,10 @@ app: - E2E_REPORT_COMMAND_TIMEOUT_SECONDS: "300" - E2E_RUBY_INSTALL_TIMEOUT_SECONDS: "1800" +trigger_map: + - pull_request_source_branch: "*" + pipeline: e2e + pipelines: e2e: workflows: @@ -288,7 +292,7 @@ workflows: : "${E2E_REACT_NATIVE_IOS_APP_PATH:?E2E_REACT_NATIVE_IOS_APP_PATH is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}" : "${E2E_REACT_NATIVE_ANDROID_APP_PATH:?E2E_REACT_NATIVE_ANDROID_APP_PATH is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}" e2e_log "Running BrowserStack Maestro for matrix row ${run_index}" - e2e/scripts/browserstack_maestro \ + E2E_STRICT="${E2E_STRICT:-false}" e2e/scripts/browserstack_maestro \ --index "$run_index" \ --suite-zip "$E2E_MAESTRO_SUITE_ZIP" \ --output-dir "$results_dir" diff --git a/e2e/scripts/browserstack_maestro b/e2e/scripts/browserstack_maestro index d31f1547..a07322e5 100755 --- a/e2e/scripts/browserstack_maestro +++ b/e2e/scripts/browserstack_maestro @@ -33,11 +33,15 @@ class BrowserStackMaestro sessions = fetch_sessions(build_status) result = normalize_result(run, device, app, suite, build, build_status, sessions) write_json("result.json", result) - exit(result.fetch("passed") ? 0 : 1) + exit(result.fetch("passed") || !strict? ? 0 : 1) end private + def strict? + ENV.fetch("E2E_STRICT", "false") == "true" + end + def resolve_device(run) override = ENV["E2E_DEVICE_OVERRIDE"] || ENV["E2E_#{run.fetch("platform").upcase}_DEVICE_OVERRIDE"] if override && !override.empty? @@ -85,12 +89,21 @@ class BrowserStackMaestro response = get_json("/app-automate/maestro/v2/builds/#{build_id}") write_json("build-status.json", response) return response if TERMINAL_STATUSES.include?(response.fetch("status").to_s.downcase) - raise "BrowserStack build timed out: #{build_id}" if Time.now >= deadline + if Time.now >= deadline + stop_build(build_id) + raise "BrowserStack build timed out: #{build_id}" + end sleep ENV.fetch("E2E_BROWSERSTACK_POLL_SECONDS", "30").to_i end end + def stop_build(build_id) + post_json("/app-automate/maestro/builds/#{build_id}/stop", {}) + rescue StandardError => error + warn "Unable to stop BrowserStack build #{build_id}: #{error.message}" + end + def fetch_sessions(build_status) build_id = build_status.fetch("id") build_status.fetch("devices", []).flat_map do |device| @@ -155,12 +168,28 @@ class BrowserStackMaestro end def execute_request(request) - request.basic_auth(@username, @access_key) - response = Net::HTTP.start(API_HOST, 443, use_ssl: true) { |http| http.request(request) } - parsed = JSON.parse(response.body) - return parsed if response.is_a?(Net::HTTPSuccess) + attempts = 0 + retries = ENV.fetch("E2E_BROWSERSTACK_API_RETRIES", "1").to_i + loop do + attempts += 1 + request.basic_auth(@username, @access_key) + response = Net::HTTP.start(API_HOST, 443, use_ssl: true) { |http| http.request(request) } + parsed = parse_response_body(response) + return parsed if response.is_a?(Net::HTTPSuccess) + raise "BrowserStack request failed #{response.code}: #{parsed}" unless retryable_browserstack_response?(response) && attempts <= retries + + sleep attempts + end + end + + def parse_response_body(response) + JSON.parse(response.body) + rescue JSON::ParserError + raise "BrowserStack request returned non-JSON response" + end - raise "BrowserStack request failed #{response.code}: #{parsed}" + def retryable_browserstack_response?(response) + response.code.to_i == 429 || response.code.to_i >= 500 end def custom_id(run, suffix) diff --git a/e2e/test/soft_launch_test.rb b/e2e/test/soft_launch_test.rb new file mode 100644 index 00000000..6b3f4fa2 --- /dev/null +++ b/e2e/test/soft_launch_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "yaml" + +class E2ESoftLaunchTest < Minitest::Test + BITRISE_CONFIG_PATH = "e2e/bitrise.yml" + + def test_bitrise_pr_trigger_runs_e2e_pipeline + config = YAML.safe_load_file(BITRISE_CONFIG_PATH) + trigger = config.fetch("trigger_map").find { |entry| entry["pull_request_source_branch"] == "*" } + + assert_equal "e2e", trigger.fetch("pipeline") + end + + def test_browserstack_runner_supports_soft_fail_and_retry_controls + script = File.read("e2e/scripts/browserstack_maestro") + + assert_includes script, "E2E_STRICT" + assert_includes script, "E2E_BROWSERSTACK_API_RETRIES" + assert_includes script, "retryable_browserstack_response?" + assert_includes script, '/app-automate/maestro/builds/#{build_id}/stop' + end + + def test_bitrise_browserstack_workflow_defaults_to_soft_fail + config = YAML.safe_load_file(BITRISE_CONFIG_PATH) + run_script = workflow_scripts(config, "e2e-run-browserstack") + + assert_includes run_script, "E2E_STRICT=\"${E2E_STRICT:-false}\"" + end + + def test_runbook_exists + assert File.exist?("e2e/RUNBOOK.md") + end + + private + + def workflow_scripts(config, workflow) + config.fetch("workflows").fetch(workflow).fetch("steps").flat_map do |step| + step_value = step.values.first + next [] unless step.keys.first.start_with?("script") + + step_value.fetch("inputs", []).filter_map { |input| input["content"] } + end.join("\n") + end +end From b93f686d614b84c949e690a60dfd3165567ba8f7 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Wed, 1 Jul 2026 22:00:33 +0100 Subject: [PATCH 2/2] Fix Android release/e2e white screen by matching bundle asset name Assisted-By: devx/6c1e3ad5-96c8-4972-b087-da7ff7b195c3 --- .../com/shopify/checkoutkit/reactnativedemo/MainApplication.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platforms/react-native/sample/android/app/src/main/java/com/shopify/checkoutkit/reactnativedemo/MainApplication.kt b/platforms/react-native/sample/android/app/src/main/java/com/shopify/checkoutkit/reactnativedemo/MainApplication.kt index 3ebe2007..223c96ed 100644 --- a/platforms/react-native/sample/android/app/src/main/java/com/shopify/checkoutkit/reactnativedemo/MainApplication.kt +++ b/platforms/react-native/sample/android/app/src/main/java/com/shopify/checkoutkit/reactnativedemo/MainApplication.kt @@ -22,6 +22,8 @@ class MainApplication : Application(), ReactApplication { override fun getJSMainModuleName(): String = "index" + override fun getBundleAssetName(): String = "CheckoutKitReactNativeDemo.android.bundle" + override fun getUseDeveloperSupport(): Boolean = BuildConfig.DEBUG override val isHermesEnabled: Boolean = BuildConfig.IS_HERMES_ENABLED