diff --git a/e2e/BITRISE.md b/e2e/BITRISE.md index ca630aa6..49a0c3e8 100644 --- a/e2e/BITRISE.md +++ b/e2e/BITRISE.md @@ -75,11 +75,9 @@ Secrets still need to be configured in Bitrise.io. | `STOREFRONT_DOMAIN` | Required by `scripts/setup_storefront_env` for sample app builds. | | `STOREFRONT_ACCESS_TOKEN` | Required by `scripts/setup_storefront_env` for sample app builds. | -## Future secrets +## BrowserStack secrets -Do not add BrowserStack credentials until the BrowserStack integration phase. - -Future secret names: +BrowserStack credentials are required by the `e2e-run-browserstack` workflow. | Secret | Purpose | |---|---| @@ -98,6 +96,17 @@ The React Native iOS artifact workflow overrides the default Linux stack and run Upload the required signing certificate and provisioning profile for the React Native sample app to the Bitrise app before running the iOS artifact workflow. The default provisioning profile specifier is `bitrise-checkout-kit-e2e`; update `E2E_IOS_PROVISIONING_PROFILE_SPECIFIER` in `e2e/bitrise.yml` if the Bitrise-installed profile uses a different name. +## BrowserStack execution + +The BrowserStack workflow resolves the Bitrise parallel index into an E2E matrix row, resolves a BrowserStack device dynamically, uploads the app artifact and Maestro suite zip, runs the selected Maestro flow, and stores raw plus normalized result JSON as artifacts. + +The initial launch smoke sends only non-sensitive Maestro environment values to BrowserStack: + +- `E2E_APP_ID` +- `E2E_READY_MARKER` + +Do not pass storefront tokens or customer data through BrowserStack Maestro environment variables without explicit review, because those values are visible in BrowserStack dashboards. + ## Caching React Native Android E2E builds use the released native Maven artifact versions declared by the React Native sample and module configuration. Do not set local native SDK override flags for these builds. diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 5a0927de..54538087 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -50,7 +50,7 @@ workflows: - key: |- e2e-ruby-linux-{{ checksum ".ruby-version" }} - script@1: - title: Validate E2E matrix and create suite placeholder + title: Validate E2E matrix and package Maestro suite inputs: - content: |- set -euo pipefail @@ -59,11 +59,12 @@ workflows: e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" ruby e2e/scripts/e2e_matrix validate e2e_log "Expanding E2E matrix" e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" ruby e2e/scripts/e2e_matrix expand > "$BITRISE_DEPLOY_DIR/e2e-matrix.json" - e2e_log "Creating Maestro suite placeholder" + e2e_log "Packaging Maestro suite" mkdir -p "$BITRISE_DEPLOY_DIR/e2e" - echo "Phase 2 placeholder for BrowserStack Maestro suite zip" > "$BITRISE_DEPLOY_DIR/e2e/maestro-suite.zip" + suite_zip="$BITRISE_DEPLOY_DIR/e2e/maestro-suite.zip" + e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" e2e/scripts/package_maestro_suite --output "$suite_zip" e2e_log "Publishing Maestro suite path" - envman add --key E2E_MAESTRO_SUITE_ZIP --value "$BITRISE_DEPLOY_DIR/e2e/maestro-suite.zip" + envman add --key E2E_MAESTRO_SUITE_ZIP --value "$suite_zip" - save-cache@1: is_always_run: true inputs: @@ -267,20 +268,29 @@ workflows: e2e-ruby-linux-{{ checksum ".ruby-version" }} - pull-intermediate-files@1: inputs: - - artifact_sources: |- - e2e-package-suite - e2e-build-react-native-* + - artifact_sources: e2e-package-suite,e2e-build-react-native-.* - script@1: - title: Resolve E2E matrix row placeholder + title: Run BrowserStack Maestro for matrix row inputs: - content: |- set -euo pipefail source e2e/scripts/bitrise_ci_helpers - e2e_prepare_ruby run_index="${BITRISE_IO_PARALLEL_INDEX:-0}" - mkdir -p "$BITRISE_DEPLOY_DIR/e2e/results" - e2e_log "Resolving E2E matrix row ${run_index}" - e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" ruby e2e/scripts/e2e_matrix expand --index "$run_index" > "$BITRISE_DEPLOY_DIR/e2e/results/run-${run_index}.json" + results_root="$BITRISE_DEPLOY_DIR/e2e/results" + results_dir="$results_root/run-${run_index}" + mkdir -p "$results_dir" + envman add --key E2E_BROWSERSTACK_RESULTS_DIR --value "$results_dir" + e2e_prepare_ruby + : "${BROWSERSTACK_USERNAME:?BROWSERSTACK_USERNAME is required}" + : "${BROWSERSTACK_ACCESS_KEY:?BROWSERSTACK_ACCESS_KEY is required}" + : "${E2E_MAESTRO_SUITE_ZIP:?E2E_MAESTRO_SUITE_ZIP is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}" + : "${E2E_REACT_NATIVE_IOS_APP_PATH:?E2E_REACT_NATIVE_IOS_APP_PATH is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}" + : "${E2E_REACT_NATIVE_ANDROID_APP_PATH:?E2E_REACT_NATIVE_ANDROID_APP_PATH is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}" + e2e_log "Running BrowserStack Maestro for matrix row ${run_index}" + e2e/scripts/browserstack_maestro \ + --index "$run_index" \ + --suite-zip "$E2E_MAESTRO_SUITE_ZIP" \ + --output-dir "$results_dir" - save-cache@1: is_always_run: true inputs: diff --git a/e2e/lib/browserstack_device_resolver.rb b/e2e/lib/browserstack_device_resolver.rb new file mode 100644 index 00000000..6daf87a4 --- /dev/null +++ b/e2e/lib/browserstack_device_resolver.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +class BrowserStackDeviceResolver + def initialize(devices, limits = []) + @devices = devices + @limits = limits + end + + def resolve(selector) + platform, form_factor, track = selector.split(":") + raise ArgumentError, "unsupported device selector: #{selector}" unless platform && form_factor == "phone" && track + + candidates = available_phone_devices(platform) + os_version = os_version_for_track(candidates, track) + device = candidates.select { |candidate| candidate.fetch("os_version") == os_version } + .sort_by { |candidate| candidate.fetch("device") } + .first + + raise ArgumentError, "no BrowserStack device available for #{selector}" unless device + + { + "device_selector" => selector, + "platform" => platform, + "os_track" => track, + "resolved_device" => device.fetch("device"), + "resolved_os_version" => device.fetch("os_version"), + "browserstack_device" => "#{device.fetch("device")}-#{device.fetch("os_version")}" + } + end + + private + + def available_phone_devices(platform) + @devices.select do |device| + device.fetch("os") == platform && + device.fetch("realMobile", true) && + phone?(device.fetch("device")) && + available?(device) + end + end + + def phone?(device_name) + !device_name.match?(/ipad|tablet| tab\b/i) + end + + def available?(device) + limit = @limits.find do |candidate| + candidate.fetch("os") == device.fetch("os") && + candidate.fetch("os_version") == device.fetch("os_version") && + candidate.fetch("device") == device.fetch("device") + end + + return true unless limit + + limit.fetch("group_usage", 0).to_i < limit.fetch("device_limit", 1).to_i + end + + def os_version_for_track(candidates, track) + versions = candidates.map { |candidate| candidate.fetch("os_version") }.uniq.sort_by { |version| version_segments(version) } + raise ArgumentError, "no BrowserStack devices available" if versions.empty? + + case track + when "latest" + versions.last + when "previous" + major_versions = versions.group_by { |version| version_segments(version).first }.keys.sort + previous_major = major_versions[-2] + raise ArgumentError, "no previous BrowserStack OS version available" unless previous_major + + versions.select { |version| version_segments(version).first == previous_major }.last + else + raise ArgumentError, "unsupported OS track: #{track}" + end + end + + def version_segments(version) + version.to_s.split(".").map(&:to_i) + end +end diff --git a/e2e/scripts/browserstack_maestro b/e2e/scripts/browserstack_maestro new file mode 100755 index 00000000..d31f1547 --- /dev/null +++ b/e2e/scripts/browserstack_maestro @@ -0,0 +1,205 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "base64" +require "fileutils" +require "json" +require "net/http" +require "optparse" +require "securerandom" +require "uri" +require_relative "../lib/browserstack_device_resolver" +require_relative "../lib/e2e_matrix" + +class BrowserStackMaestro + API_HOST = "api-cloud.browserstack.com" + TERMINAL_STATUSES = %w[passed failed error timedout stopped done].freeze + + def initialize(options) + @options = options + @username = ENV.fetch("BROWSERSTACK_USERNAME") + @access_key = ENV.fetch("BROWSERSTACK_ACCESS_KEY") + end + + def run + FileUtils.mkdir_p(output_dir) + run = E2EMatrix.load(matrix_path).run_at(run_index) + app_path = ENV.fetch(run.fetch("artifact_env")) + device = resolve_device(run) + app = upload_file("/app-automate/maestro/v2/app", app_path, custom_id(run, "app")) + suite = upload_file("/app-automate/maestro/v2/test-suite", suite_zip, custom_id(run, "suite")) + build = start_build(run, app.fetch("app_url"), suite.fetch("test_suite_url"), device.fetch("browserstack_device")) + build_status = poll_build(build.fetch("build_id")) + sessions = fetch_sessions(build_status) + result = normalize_result(run, device, app, suite, build, build_status, sessions) + write_json("result.json", result) + exit(result.fetch("passed") ? 0 : 1) + end + + private + + def resolve_device(run) + override = ENV["E2E_DEVICE_OVERRIDE"] || ENV["E2E_#{run.fetch("platform").upcase}_DEVICE_OVERRIDE"] + if override && !override.empty? + return { + "device_selector" => run.fetch("device_selector"), + "browserstack_device" => override, + "resolved_device" => override.split("-").first, + "resolved_os_version" => override.split("-").last + } + end + + devices = get_json("/app-automate/devices.json") + limits = get_json("/app-automate/device_tier_limits.json") + BrowserStackDeviceResolver.new(devices, limits).resolve(run.fetch("device_selector")) + end + + def upload_file(path, file_path, custom_id) + response = multipart_post(path, file_path, custom_id) + write_json("#{custom_id}.json", response) + response + end + + def start_build(run, app_url, test_suite_url, device) + body = { + app: app_url, + testSuite: test_suite_url, + project: ENV.fetch("E2E_BROWSERSTACK_PROJECT", "checkout-kit-e2e"), + buildTag: ENV.fetch("BITRISE_GIT_COMMIT", "local"), + customBuildName: run.fetch("id"), + devices: [device], + execute: [run.fetch("execute")], + setEnvVariables: { + E2E_APP_ID: run.fetch("app_id"), + E2E_READY_MARKER: run.fetch("ready_marker") + } + } + response = post_json("/app-automate/maestro/v2/#{run.fetch("platform")}/build", body) + write_json("build-start.json", response) + response + end + + def poll_build(build_id) + deadline = Time.now + ENV.fetch("E2E_BROWSERSTACK_TIMEOUT_SECONDS", "1800").to_i + loop do + response = get_json("/app-automate/maestro/v2/builds/#{build_id}") + write_json("build-status.json", response) + return response if TERMINAL_STATUSES.include?(response.fetch("status").to_s.downcase) + raise "BrowserStack build timed out: #{build_id}" if Time.now >= deadline + + sleep ENV.fetch("E2E_BROWSERSTACK_POLL_SECONDS", "30").to_i + end + end + + def fetch_sessions(build_status) + build_id = build_status.fetch("id") + build_status.fetch("devices", []).flat_map do |device| + device.fetch("sessions", []).map do |session| + get_json("/app-automate/maestro/v2/builds/#{build_id}/sessions/#{session.fetch("id")}") + end + end + end + + def normalize_result(run, device, app, suite, build, build_status, sessions) + status = build_status.fetch("status").to_s.downcase + failed_tests = sessions.flat_map do |session| + session.dig("testcases", "data").to_a.flat_map do |group| + group.fetch("testcases", []).select { |testcase| testcase.fetch("status", "") != "passed" } + end + end + + { + "id" => run.fetch("id"), + "status_context" => run.fetch("status_context"), + "platform" => run.fetch("platform"), + "target" => run.fetch("target"), + "os_track" => run.fetch("os_track"), + "execute" => run.fetch("execute"), + "device_selector" => device.fetch("device_selector"), + "resolved_device" => device.fetch("browserstack_device"), + "app_url" => app.fetch("app_url"), + "test_suite_url" => suite.fetch("test_suite_url"), + "build_id" => build.fetch("build_id"), + "status" => status, + "passed" => status == "passed" && failed_tests.empty?, + "failed_tests" => failed_tests + } + end + + def multipart_post(path, file_path, custom_id) + boundary = "----checkout-kit-#{SecureRandom.hex(12)}" + file = File.binread(file_path) + body = +"" + body << "--#{boundary}\r\n" + body << "Content-Disposition: form-data; name=\"file\"; filename=\"#{File.basename(file_path)}\"\r\n\r\n" + body << file + body << "\r\n--#{boundary}\r\n" + body << "Content-Disposition: form-data; name=\"custom_id\"\r\n\r\n" + body << custom_id + body << "\r\n--#{boundary}--\r\n" + request = Net::HTTP::Post.new(path) + request["Content-Type"] = "multipart/form-data; boundary=#{boundary}" + request.body = body + execute_request(request) + end + + def get_json(path) + execute_request(Net::HTTP::Get.new(path)) + end + + def post_json(path, body) + request = Net::HTTP::Post.new(path) + request["Content-Type"] = "application/json" + request.body = JSON.generate(body) + execute_request(request) + end + + def execute_request(request) + request.basic_auth(@username, @access_key) + response = Net::HTTP.start(API_HOST, 443, use_ssl: true) { |http| http.request(request) } + parsed = JSON.parse(response.body) + return parsed if response.is_a?(Net::HTTPSuccess) + + raise "BrowserStack request failed #{response.code}: #{parsed}" + end + + def custom_id(run, suffix) + ["checkout-kit", run.fetch("id"), ENV.fetch("BITRISE_GIT_COMMIT", "local"), suffix].join("-").gsub(/[^A-Za-z0-9._-]/, "-")[0, 100] + end + + def write_json(name, object) + File.write(File.join(output_dir, name), JSON.pretty_generate(object)) + end + + def output_dir + @options.fetch(:output_dir) + end + + def matrix_path + @options.fetch(:matrix_path) + end + + def run_index + @options.fetch(:run_index) + end + + def suite_zip + @options.fetch(:suite_zip) + end +end + +options = { + matrix_path: "e2e/config/matrix.yml", + run_index: Integer(ENV.fetch("BITRISE_IO_PARALLEL_INDEX", "0")), + suite_zip: ENV.fetch("E2E_MAESTRO_SUITE_ZIP"), + output_dir: ENV.fetch("E2E_BROWSERSTACK_RESULTS_DIR", File.join(Dir.pwd, "e2e-results")) +} + +OptionParser.new do |opts| + opts.on("--matrix PATH") { |path| options[:matrix_path] = path } + opts.on("--index INDEX", Integer) { |index| options[:run_index] = index } + opts.on("--suite-zip PATH") { |path| options[:suite_zip] = path } + opts.on("--output-dir PATH") { |path| options[:output_dir] = path } +end.parse! + +BrowserStackMaestro.new(options).run diff --git a/e2e/scripts/package_maestro_suite b/e2e/scripts/package_maestro_suite new file mode 100755 index 00000000..fa36ed6d --- /dev/null +++ b/e2e/scripts/package_maestro_suite @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +output="" +parent="checkout-kit-maestro-suite" + +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + output="$2" + shift 2 + ;; + *) + echo "Usage: e2e/scripts/package_maestro_suite --output PATH" >&2 + exit 1 + ;; + esac +done + +if [ -z "$output" ]; then + echo "Usage: e2e/scripts/package_maestro_suite --output PATH" >&2 + exit 1 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/checkout-kit-maestro-suite.XXXXXX")" +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +suite_root="$tmpdir/$parent" +mkdir -p "$suite_root" + +if [ -d "$repo_root/e2e/tests" ]; then + cp -R "$repo_root/e2e/tests" "$suite_root/tests" +fi + +if [ -d "$repo_root/e2e/flows" ]; then + cp -R "$repo_root/e2e/flows" "$suite_root/flows" +fi + +if [ -f "$repo_root/e2e/config.yaml" ]; then + cp "$repo_root/e2e/config.yaml" "$suite_root/config.yaml" +fi + +mkdir -p "$(dirname "$output")" +rm -f "$output" +( + cd "$tmpdir" + zip -qr "$output" "$parent" +) + +echo "$output" diff --git a/e2e/test/bitrise_pipeline_test.rb b/e2e/test/bitrise_pipeline_test.rb index 601b9f43..4f4a9b51 100644 --- a/e2e/test/bitrise_pipeline_test.rb +++ b/e2e/test/bitrise_pipeline_test.rb @@ -44,12 +44,28 @@ def test_default_e2e_stack_is_linux_android assert_equal "g2.linux.medium", meta.fetch("machine_type_id") end - def test_pipeline_skeleton_does_not_call_browserstack_yet - bitrise_yml = File.read(BITRISE_CONFIG_PATH) - - refute_includes bitrise_yml, "api-cloud.browserstack.com" - refute_includes bitrise_yml, "BROWSERSTACK_USERNAME" - refute_includes bitrise_yml, "BROWSERSTACK_ACCESS_KEY" + def test_browserstack_workflow_runs_real_maestro_integration + package_script = workflow_script("e2e-package-suite") + run_script = workflow_script("e2e-run-browserstack") + + assert_includes package_script, 'e2e_log "Packaging Maestro suite"' + assert_includes package_script, 'e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" e2e/scripts/package_maestro_suite --output "$suite_zip"' + assert_includes package_script, "e2e/scripts/package_maestro_suite" + refute_includes package_script, "Phase 2 placeholder for BrowserStack Maestro suite zip" + pull_step = workflow_step(load_bitrise_config, "e2e-run-browserstack", "pull-intermediate-files") + assert_equal "e2e-package-suite,e2e-build-react-native-.*", pull_step.fetch("inputs").find { |input| input.key?("artifact_sources") }.fetch("artifact_sources") + assert_includes run_script, 'e2e_log "Running BrowserStack Maestro for matrix row ${run_index}"' + assert_includes run_script, 'results_root="$BITRISE_DEPLOY_DIR/e2e/results"' + assert_includes run_script, 'mkdir -p "$results_dir"' + assert_command_order run_script, 'mkdir -p "$results_dir"', 'BROWSERSTACK_USERNAME' + assert_includes run_script, 'E2E_MAESTRO_SUITE_ZIP is required' + assert_includes run_script, 'E2E_REACT_NATIVE_IOS_APP_PATH is required' + assert_includes run_script, 'E2E_REACT_NATIVE_ANDROID_APP_PATH is required' + assert_command_order run_script, 'E2E_MAESTRO_SUITE_ZIP is required', 'e2e/scripts/browserstack_maestro' + assert_includes run_script, "e2e/scripts/browserstack_maestro" + assert_includes run_script, "BROWSERSTACK_USERNAME" + assert_includes run_script, "BROWSERSTACK_ACCESS_KEY" + refute_includes run_script, "Resolve E2E matrix row placeholder" end def test_pipeline_uses_intermediate_file_sharing_placeholders @@ -115,7 +131,7 @@ def test_package_suite_logs_and_bounds_each_command assert_includes script, 'e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" ruby e2e/scripts/e2e_matrix validate' assert_includes script, 'e2e_log "Expanding E2E matrix"' - assert_includes script, 'e2e_log "Creating Maestro suite placeholder"' + assert script.include?('e2e_log "Creating Maestro suite placeholder"') || script.include?('e2e_log "Packaging Maestro suite"') assert_includes script, 'e2e_log "Publishing Maestro suite path"' end diff --git a/e2e/test/browserstack_device_resolver_test.rb b/e2e/test/browserstack_device_resolver_test.rb new file mode 100644 index 00000000..41a8448c --- /dev/null +++ b/e2e/test/browserstack_device_resolver_test.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../lib/browserstack_device_resolver" + +class BrowserStackDeviceResolverTest < Minitest::Test + def test_resolves_latest_ios_phone + resolver = BrowserStackDeviceResolver.new(devices, limits) + + device = resolver.resolve("ios:phone:latest") + + assert_equal "iPhone 16-18.0", device.fetch("browserstack_device") + assert_equal "ios:phone:latest", device.fetch("device_selector") + end + + def test_resolves_previous_android_phone + resolver = BrowserStackDeviceResolver.new(devices, limits) + + device = resolver.resolve("android:phone:previous") + + assert_equal "Google Pixel 8-14.0", device.fetch("browserstack_device") + end + + def test_ignores_devices_at_capacity + resolver = BrowserStackDeviceResolver.new(devices, limits) + + device = resolver.resolve("android:phone:latest") + + assert_equal "Samsung Galaxy S24-15.0", device.fetch("browserstack_device") + end + + private + + def devices + [ + {"os" => "ios", "os_version" => "17.0", "device" => "iPhone 15", "realMobile" => true}, + {"os" => "ios", "os_version" => "18.0", "device" => "iPhone 16", "realMobile" => true}, + {"os" => "ios", "os_version" => "18.0", "device" => "iPad Pro", "realMobile" => true}, + {"os" => "android", "os_version" => "14.0", "device" => "Google Pixel 8", "realMobile" => true}, + {"os" => "android", "os_version" => "15.0", "device" => "Google Pixel 9", "realMobile" => true}, + {"os" => "android", "os_version" => "15.0", "device" => "Samsung Galaxy S24", "realMobile" => true} + ] + end + + def limits + [ + {"os" => "android", "os_version" => "15.0", "device" => "Google Pixel 9", "device_limit" => 1, "group_usage" => 1}, + {"os" => "android", "os_version" => "15.0", "device" => "Samsung Galaxy S24", "device_limit" => 1, "group_usage" => 0} + ] + end +end diff --git a/e2e/test/maestro_suite_package_test.rb b/e2e/test/maestro_suite_package_test.rb new file mode 100644 index 00000000..198d4b31 --- /dev/null +++ b/e2e/test/maestro_suite_package_test.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "shellwords" +require "tmpdir" + +class MaestroSuitePackageTest < Minitest::Test + def test_packages_suite_with_single_parent_folder + Dir.mktmpdir do |dir| + output = File.join(dir, "suite.zip") + + system("e2e/scripts/package_maestro_suite", "--output", output, exception: true) + + entries = `unzip -Z1 #{Shellwords.escape(output)}`.split("\n") + roots = entries.map { |entry| entry.split("/").first }.uniq + + assert_equal ["checkout-kit-maestro-suite"], roots + assert_includes entries, "checkout-kit-maestro-suite/tests/shared/launch-smoke.yaml" + refute_includes entries, "checkout-kit-maestro-suite/config/matrix.yml" + refute_includes entries, "checkout-kit-maestro-suite/test/e2e_matrix_test.rb" + end + end +end