Run BrowserStack Maestro from E2E matrix rows

e2e/BITRISE.md

@@ -75,11 +75,9 @@ Secrets still need to be configured in Bitrise.io.
 | `STOREFRONT_DOMAIN` | Required by `scripts/setup_storefront_env` for sample app builds. |
 | `STOREFRONT_ACCESS_TOKEN` | Required by `scripts/setup_storefront_env` for sample app builds. |
 
-## Future secrets
+## BrowserStack secrets
 
-Do not add BrowserStack credentials until the BrowserStack integration phase.
-
-Future secret names:
+BrowserStack credentials are required by the `e2e-run-browserstack` workflow.
 
 | Secret | Purpose |
 |---|---|
@@ -98,6 +96,17 @@ The React Native iOS artifact workflow overrides the default Linux stack and run
 
 Upload the required signing certificate and provisioning profile for the React Native sample app to the Bitrise app before running the iOS artifact workflow. The default provisioning profile specifier is `bitrise-checkout-kit-e2e`; update `E2E_IOS_PROVISIONING_PROFILE_SPECIFIER` in `e2e/bitrise.yml` if the Bitrise-installed profile uses a different name.
 
+## BrowserStack execution
+
+The BrowserStack workflow resolves the Bitrise parallel index into an E2E matrix row, resolves a BrowserStack device dynamically, uploads the app artifact and Maestro suite zip, runs the selected Maestro flow, and stores raw plus normalized result JSON as artifacts.
+
+The initial launch smoke sends only non-sensitive Maestro environment values to BrowserStack:
+
+- `E2E_APP_ID`
+- `E2E_READY_MARKER`
+
+Do not pass storefront tokens or customer data through BrowserStack Maestro environment variables without explicit review, because those values are visible in BrowserStack dashboards.
+
 ## Caching
 
 React Native Android E2E builds use the released native Maven artifact versions declared by the React Native sample and module configuration. Do not set local native SDK override flags for these builds.

e2e/bitrise.yml

@@ -50,7 +50,7 @@ workflows:
             - key: |-
                 e2e-ruby-linux-{{ checksum ".ruby-version" }}
       - script@1:
-          title: Validate E2E matrix and create suite placeholder
+          title: Validate E2E matrix and package Maestro suite
           inputs:
             - content: |-
                 set -euo pipefail
@@ -59,11 +59,12 @@ workflows:
                 e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" ruby e2e/scripts/e2e_matrix validate
                 e2e_log "Expanding E2E matrix"
                 e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" ruby e2e/scripts/e2e_matrix expand > "$BITRISE_DEPLOY_DIR/e2e-matrix.json"
-                e2e_log "Creating Maestro suite placeholder"
+                e2e_log "Packaging Maestro suite"
                 mkdir -p "$BITRISE_DEPLOY_DIR/e2e"
-                echo "Phase 2 placeholder for BrowserStack Maestro suite zip" > "$BITRISE_DEPLOY_DIR/e2e/maestro-suite.zip"
+                suite_zip="$BITRISE_DEPLOY_DIR/e2e/maestro-suite.zip"
+                e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" e2e/scripts/package_maestro_suite --output "$suite_zip"
                 e2e_log "Publishing Maestro suite path"
-                envman add --key E2E_MAESTRO_SUITE_ZIP --value "$BITRISE_DEPLOY_DIR/e2e/maestro-suite.zip"
+                envman add --key E2E_MAESTRO_SUITE_ZIP --value "$suite_zip"
       - save-cache@1:
           is_always_run: true
           inputs:
@@ -267,20 +268,29 @@ workflows:
                 e2e-ruby-linux-{{ checksum ".ruby-version" }}
       - pull-intermediate-files@1:
           inputs:
-            - artifact_sources: |-
-                e2e-package-suite
-                e2e-build-react-native-*
+            - artifact_sources: e2e-package-suite,e2e-build-react-native-.*
       - script@1:
-          title: Resolve E2E matrix row placeholder
+          title: Run BrowserStack Maestro for matrix row
           inputs:
             - content: |-
                 set -euo pipefail
                 source e2e/scripts/bitrise_ci_helpers
-                e2e_prepare_ruby
                 run_index="${BITRISE_IO_PARALLEL_INDEX:-0}"
-                mkdir -p "$BITRISE_DEPLOY_DIR/e2e/results"
-                e2e_log "Resolving E2E matrix row ${run_index}"
-                e2e_run_with_timeout "${E2E_PACKAGE_COMMAND_TIMEOUT_SECONDS:-300}" ruby e2e/scripts/e2e_matrix expand --index "$run_index" > "$BITRISE_DEPLOY_DIR/e2e/results/run-${run_index}.json"
+                results_root="$BITRISE_DEPLOY_DIR/e2e/results"
+                results_dir="$results_root/run-${run_index}"
+                mkdir -p "$results_dir"
+                envman add --key E2E_BROWSERSTACK_RESULTS_DIR --value "$results_dir"
+                e2e_prepare_ruby
+                : "${BROWSERSTACK_USERNAME:?BROWSERSTACK_USERNAME is required}"
+                : "${BROWSERSTACK_ACCESS_KEY:?BROWSERSTACK_ACCESS_KEY is required}"
+                : "${E2E_MAESTRO_SUITE_ZIP:?E2E_MAESTRO_SUITE_ZIP is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}"
+                : "${E2E_REACT_NATIVE_IOS_APP_PATH:?E2E_REACT_NATIVE_IOS_APP_PATH is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}"
+                : "${E2E_REACT_NATIVE_ANDROID_APP_PATH:?E2E_REACT_NATIVE_ANDROID_APP_PATH is required. Check e2e-run-browserstack pull-intermediate-files artifact_sources.}"
+                e2e_log "Running BrowserStack Maestro for matrix row ${run_index}"
+                e2e/scripts/browserstack_maestro \
+                  --index "$run_index" \
+                  --suite-zip "$E2E_MAESTRO_SUITE_ZIP" \
+                  --output-dir "$results_dir"
       - save-cache@1:
           is_always_run: true
           inputs:

e2e/lib/browserstack_device_resolver.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+class BrowserStackDeviceResolver
+  def initialize(devices, limits = [])
+    @devices = devices
+    @limits = limits
+  end
+
+  def resolve(selector)
+    platform, form_factor, track = selector.split(":")
+    raise ArgumentError, "unsupported device selector: #{selector}" unless platform && form_factor == "phone" && track
+
+    candidates = available_phone_devices(platform)
+    os_version = os_version_for_track(candidates, track)
+    device = candidates.select { |candidate| candidate.fetch("os_version") == os_version }
+      .sort_by { |candidate| candidate.fetch("device") }
+      .first
+
+    raise ArgumentError, "no BrowserStack device available for #{selector}" unless device
+
+    {
+      "device_selector" => selector,
+      "platform" => platform,
+      "os_track" => track,
+      "resolved_device" => device.fetch("device"),
+      "resolved_os_version" => device.fetch("os_version"),
+      "browserstack_device" => "#{device.fetch("device")}-#{device.fetch("os_version")}" 
+    }
+  end
+
+  private
+
+  def available_phone_devices(platform)
+    @devices.select do |device|
+      device.fetch("os") == platform &&
+        device.fetch("realMobile", true) &&
+        phone?(device.fetch("device")) &&
+        available?(device)
+    end
+  end
+
+  def phone?(device_name)
+    !device_name.match?(/ipad|tablet| tab\b/i)
+  end
+
+  def available?(device)
+    limit = @limits.find do |candidate|
+      candidate.fetch("os") == device.fetch("os") &&
+        candidate.fetch("os_version") == device.fetch("os_version") &&
+        candidate.fetch("device") == device.fetch("device")
+    end
+
+    return true unless limit
+
+    limit.fetch("group_usage", 0).to_i < limit.fetch("device_limit", 1).to_i
+  end
+
+  def os_version_for_track(candidates, track)
+    versions = candidates.map { |candidate| candidate.fetch("os_version") }.uniq.sort_by { |version| version_segments(version) }
+    raise ArgumentError, "no BrowserStack devices available" if versions.empty?
+
+    case track
+    when "latest"
+      versions.last
+    when "previous"
+      major_versions = versions.group_by { |version| version_segments(version).first }.keys.sort
+      previous_major = major_versions[-2]
+      raise ArgumentError, "no previous BrowserStack OS version available" unless previous_major
+
+      versions.select { |version| version_segments(version).first == previous_major }.last
+    else
+      raise ArgumentError, "unsupported OS track: #{track}"
+    end
+  end
+
+  def version_segments(version)
+    version.to_s.split(".").map(&:to_i)
+  end
+end

e2e/scripts/browserstack_maestro

@@ -0,0 +1,205 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "base64"
+require "fileutils"
+require "json"
+require "net/http"
+require "optparse"
+require "securerandom"
+require "uri"
+require_relative "../lib/browserstack_device_resolver"
+require_relative "../lib/e2e_matrix"
+
+class BrowserStackMaestro
+  API_HOST = "api-cloud.browserstack.com"
+  TERMINAL_STATUSES = %w[passed failed error timedout stopped done].freeze
+
+  def initialize(options)
+    @options = options
+    @username = ENV.fetch("BROWSERSTACK_USERNAME")
+    @access_key = ENV.fetch("BROWSERSTACK_ACCESS_KEY")
+  end
+
+  def run
+    FileUtils.mkdir_p(output_dir)
+    run = E2EMatrix.load(matrix_path).run_at(run_index)
+    app_path = ENV.fetch(run.fetch("artifact_env"))
+    device = resolve_device(run)
+    app = upload_file("/app-automate/maestro/v2/app", app_path, custom_id(run, "app"))
+    suite = upload_file("/app-automate/maestro/v2/test-suite", suite_zip, custom_id(run, "suite"))
+    build = start_build(run, app.fetch("app_url"), suite.fetch("test_suite_url"), device.fetch("browserstack_device"))
+    build_status = poll_build(build.fetch("build_id"))
+    sessions = fetch_sessions(build_status)
+    result = normalize_result(run, device, app, suite, build, build_status, sessions)
+    write_json("result.json", result)
+    exit(result.fetch("passed") ? 0 : 1)
+  end
+
+  private
+
+  def resolve_device(run)
+    override = ENV["E2E_DEVICE_OVERRIDE"] || ENV["E2E_#{run.fetch("platform").upcase}_DEVICE_OVERRIDE"]
+    if override && !override.empty?
+      return {
+        "device_selector" => run.fetch("device_selector"),
+        "browserstack_device" => override,
+        "resolved_device" => override.split("-").first,
+        "resolved_os_version" => override.split("-").last
+      }
+    end
+
+    devices = get_json("/app-automate/devices.json")
+    limits = get_json("/app-automate/device_tier_limits.json")
+    BrowserStackDeviceResolver.new(devices, limits).resolve(run.fetch("device_selector"))
+  end
+
+  def upload_file(path, file_path, custom_id)
+    response = multipart_post(path, file_path, custom_id)
+    write_json("#{custom_id}.json", response)
+    response
+  end
+
+  def start_build(run, app_url, test_suite_url, device)
+    body = {
+      app: app_url,
+      testSuite: test_suite_url,
+      project: ENV.fetch("E2E_BROWSERSTACK_PROJECT", "checkout-kit-e2e"),
+      buildTag: ENV.fetch("BITRISE_GIT_COMMIT", "local"),
+      customBuildName: run.fetch("id"),
+      devices: [device],
+      execute: [run.fetch("execute")],
+      setEnvVariables: {
+        E2E_APP_ID: run.fetch("app_id"),
+        E2E_READY_MARKER: run.fetch("ready_marker")
+      }
+    }
+    response = post_json("/app-automate/maestro/v2/#{run.fetch("platform")}/build", body)
+    write_json("build-start.json", response)
+    response
+  end
+
+  def poll_build(build_id)
+    deadline = Time.now + ENV.fetch("E2E_BROWSERSTACK_TIMEOUT_SECONDS", "1800").to_i
+    loop do
+      response = get_json("/app-automate/maestro/v2/builds/#{build_id}")
+      write_json("build-status.json", response)
+      return response if TERMINAL_STATUSES.include?(response.fetch("status").to_s.downcase)
+      raise "BrowserStack build timed out: #{build_id}" if Time.now >= deadline
+
+      sleep ENV.fetch("E2E_BROWSERSTACK_POLL_SECONDS", "30").to_i
+    end
+  end
+
+  def fetch_sessions(build_status)
+    build_id = build_status.fetch("id")
+    build_status.fetch("devices", []).flat_map do |device|
+      device.fetch("sessions", []).map do |session|
+        get_json("/app-automate/maestro/v2/builds/#{build_id}/sessions/#{session.fetch("id")}")
+      end
+    end
+  end
+
+  def normalize_result(run, device, app, suite, build, build_status, sessions)
+    status = build_status.fetch("status").to_s.downcase
+    failed_tests = sessions.flat_map do |session|
+      session.dig("testcases", "data").to_a.flat_map do |group|
+        group.fetch("testcases", []).select { |testcase| testcase.fetch("status", "") != "passed" }
+      end
+    end
+
+    {
+      "id" => run.fetch("id"),
+      "status_context" => run.fetch("status_context"),
+      "platform" => run.fetch("platform"),
+      "target" => run.fetch("target"),
+      "os_track" => run.fetch("os_track"),
+      "execute" => run.fetch("execute"),
+      "device_selector" => device.fetch("device_selector"),
+      "resolved_device" => device.fetch("browserstack_device"),
+      "app_url" => app.fetch("app_url"),
+      "test_suite_url" => suite.fetch("test_suite_url"),
+      "build_id" => build.fetch("build_id"),
+      "status" => status,
+      "passed" => status == "passed" && failed_tests.empty?,
+      "failed_tests" => failed_tests
+    }
+  end
+
+  def multipart_post(path, file_path, custom_id)
+    boundary = "----checkout-kit-#{SecureRandom.hex(12)}"
+    file = File.binread(file_path)
+    body = +""
+    body << "--#{boundary}\r\n"
+    body << "Content-Disposition: form-data; name=\"file\"; filename=\"#{File.basename(file_path)}\"\r\n\r\n"
+    body << file
+    body << "\r\n--#{boundary}\r\n"
+    body << "Content-Disposition: form-data; name=\"custom_id\"\r\n\r\n"
+    body << custom_id
+    body << "\r\n--#{boundary}--\r\n"
+    request = Net::HTTP::Post.new(path)
+    request["Content-Type"] = "multipart/form-data; boundary=#{boundary}"
+    request.body = body
+    execute_request(request)
+  end
+
+  def get_json(path)
+    execute_request(Net::HTTP::Get.new(path))
+  end
+
+  def post_json(path, body)
+    request = Net::HTTP::Post.new(path)
+    request["Content-Type"] = "application/json"
+    request.body = JSON.generate(body)
+    execute_request(request)
+  end
+
+  def execute_request(request)
+    request.basic_auth(@username, @access_key)
+    response = Net::HTTP.start(API_HOST, 443, use_ssl: true) { |http| http.request(request) }
+    parsed = JSON.parse(response.body)
+    return parsed if response.is_a?(Net::HTTPSuccess)
+
+    raise "BrowserStack request failed #{response.code}: #{parsed}"
+  end
+
+  def custom_id(run, suffix)
+    ["checkout-kit", run.fetch("id"), ENV.fetch("BITRISE_GIT_COMMIT", "local"), suffix].join("-").gsub(/[^A-Za-z0-9._-]/, "-")[0, 100]
+  end
+
+  def write_json(name, object)
+    File.write(File.join(output_dir, name), JSON.pretty_generate(object))
+  end
+
+  def output_dir
+    @options.fetch(:output_dir)
+  end
+
+  def matrix_path
+    @options.fetch(:matrix_path)
+  end
+
+  def run_index
+    @options.fetch(:run_index)
+  end
+
+  def suite_zip
+    @options.fetch(:suite_zip)
+  end
+end
+
+options = {
+  matrix_path: "e2e/config/matrix.yml",
+  run_index: Integer(ENV.fetch("BITRISE_IO_PARALLEL_INDEX", "0")),
+  suite_zip: ENV.fetch("E2E_MAESTRO_SUITE_ZIP"),
+  output_dir: ENV.fetch("E2E_BROWSERSTACK_RESULTS_DIR", File.join(Dir.pwd, "e2e-results"))
+}
+
+OptionParser.new do |opts|
+  opts.on("--matrix PATH") { |path| options[:matrix_path] = path }
+  opts.on("--index INDEX", Integer) { |index| options[:run_index] = index }
+  opts.on("--suite-zip PATH") { |path| options[:suite_zip] = path }
+  opts.on("--output-dir PATH") { |path| options[:output_dir] = path }
+end.parse!
+
+BrowserStackMaestro.new(options).run