Add CI job to test building on all 3 OSes, and test version output (#3275)

* CI: Fix gradle config to prevent android sentinel failures or iOS dependency failures

* Add CI job to test building on all 3 OSes, and test version output

Disable Maestro CLI analytics to not pollute the test

Fix Windows path setting in GHA

Merge e2e testing together

Make CLI tests local-first

* Extra test, improved usage and formatting

* Fix Windows tests that output to stderr

* Additional tests, deal with ANSI in bash

Author: Fishbowler

.github/workflows/test-e2e.yaml

@@ -138,17 +138,22 @@ jobs:
           fi
 
   build:
-    name: Build on Java ${{ matrix.java-version }}
+    name: Build on ${{ matrix.os }}
     needs: validate-inputs
     if: ${{ always() && (needs.validate-inputs.result == 'success' || needs.validate-inputs.result == 'skipped') }}
-    runs-on: macos-latest
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 20
 
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
         java-version: [17]
 
+    defaults:
+      run:
+        shell: bash
+
     steps:
       - name: Clone repository
         uses: actions/checkout@v6
@@ -161,19 +166,21 @@ jobs:
           cache: gradle
 
       - name: Build xctest-runner
+        if: runner.os == 'macOS'
         run: ./maestro-ios-xctest-runner/build-maestro-ios-runner.sh | xcbeautify
 
       - name: Build Maestro CLI
         run: ./gradlew :maestro-cli:distZip
 
-      - name: Upload zipped Maestro CLI artifact
+      - name: Upload Maestro CLI artifact
         uses: actions/upload-artifact@v7
         with:
-          name: maestro-cli-jdk${{ matrix.java-version }}-run_id${{ github.run_id }}
+          name: maestro-cli-${{ matrix.os }}-jdk${{ matrix.java-version }}-run_id${{ github.run_id }}
           path: maestro-cli/build/distributions/maestro.zip
           retention-days: 1
 
       - name: Upload build/Products to artifacts
+        if: runner.os == 'macOS'
         uses: actions/upload-artifact@v7
         with:
           name: build__Products-jdk${{ matrix.java-version }}
@@ -208,7 +215,7 @@ jobs:
       - name: Download artifacts
         uses: actions/download-artifact@v8
         with:
-          name: maestro-cli-jdk17-run_id${{ github.run_id }}
+          name: maestro-cli-macos-latest-jdk17-run_id${{ github.run_id }}
 
       - name: Add Maestro CLI executable to PATH
         run: |
@@ -324,7 +331,7 @@ jobs:
       - name: Download Maestro build from previous job
         uses: actions/download-artifact@v8
         with:
-          name: maestro-cli-jdk17-run_id${{ github.run_id }}
+          name: maestro-cli-macos-latest-jdk17-run_id${{ github.run_id }}
 
       - name: Add Maestro CLI executable to PATH
         run: |
@@ -435,7 +442,7 @@ jobs:
       - name: Download artifacts
         uses: actions/download-artifact@v8
         with:
-          name: maestro-cli-jdk17-run_id${{ github.run_id }}
+          name: maestro-cli-macos-latest-jdk17-run_id${{ github.run_id }}
 
       - name: Add Maestro CLI executable to PATH
         run: |
@@ -527,7 +534,7 @@ jobs:
       - name: Download Maestro artifact
         uses: actions/download-artifact@v8
         with:
-          name: maestro-cli-jdk17-run_id${{ github.run_id }}
+          name: maestro-cli-macos-latest-jdk17-run_id${{ github.run_id }}
 
       - name: Download build/Products artifact
         uses: actions/download-artifact@v8
@@ -560,3 +567,57 @@ jobs:
           path: ~/Library/Logs/maestro/xctest_runner_logs
           retention-days: 7
           include-hidden-files: true
+
+  test-cli:
+    name: Verify CLI version on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: build
+    if: ${{ always() && needs.build.result == 'success' }}
+    timeout-minutes: 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        java-version: [17]
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v6
+
+      - name: Set up Java
+        uses: actions/setup-java@v5
+        with:
+          distribution: zulu
+          java-version: ${{ matrix.java-version }}
+
+      - name: Download Maestro CLI artifact
+        uses: actions/download-artifact@v8
+        with:
+          name: maestro-cli-${{ matrix.os }}-jdk${{ matrix.java-version }}-run_id${{ github.run_id }}
+
+      - name: Add Maestro CLI to PATH (Linux / macOS)
+        if: runner.os != 'Windows'
+        run: |
+          unzip maestro.zip -d maestro_extracted
+          echo "$PWD/maestro_extracted/maestro/bin" >> $GITHUB_PATH
+
+      - name: Add Maestro CLI to PATH (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          Expand-Archive -Path maestro.zip -DestinationPath maestro_extracted
+          "$PWD\maestro_extracted\maestro\bin" | Out-File -FilePath $env:GITHUB_PATH -Append
+
+      - name: Test CLI (Linux / macOS)
+        if: runner.os != 'Windows'
+        run: e2e/cli/test-cli.sh
+
+      - name: Test CLI (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: e2e/cli/test-cli.ps1

e2e/cli/test-cli.ps1

@@ -0,0 +1,44 @@
+$env:MAESTRO_CLI_NO_ANALYTICS = 1
+$pass = 0; $fail = 0
+
+function Check([string]$desc, [string]$cmd, [string]$assertion, [string]$expected) {
+    $actual = (Invoke-Expression "$cmd 2>&1" | Out-String).Trim()
+    $ok = switch ($assertion) {
+        'equals'   { $actual -eq $expected }
+        'includes' { $actual -like "*$expected*" }
+        'excludes' { $actual -notlike "*$expected*" }
+        default    { Write-Error "unknown assertion '$assertion' (use: equals, includes, excludes)"; exit 1 }
+    }
+    if ($ok) {
+        Write-Host "PASS: $desc"
+        $script:pass++
+    } else {
+        Write-Host ("FAIL: {0}" -f $desc)
+        Write-Host ("      {0,-11} {1}" -f "cmd:",       $cmd)
+        Write-Host ("      {0,-11} {1}" -f "assertion:", $assertion)
+        Write-Host ("      {0,-11} {1}" -f "expected:",  $expected)
+        Write-Host ("      {0,-11} {1}" -f "got:",       $actual)
+        $script:fail++
+    }
+}
+
+$version = (Select-String -Path 'maestro-cli/gradle.properties' -Pattern '^CLI_VERSION=').Line `
+    -replace '^CLI_VERSION=', ''
+
+# TESTS START HERE:
+
+Check "maestro -v equals gradle.properties version" `
+    "maestro -v" "equals" $version
+Check "maestro gives usage instructions when called without parameters" `
+    "maestro" "includes" "Usage: maestro"
+Check "maestro gives usage instructions when called with --help" `
+    "maestro --help" "includes" "Usage: maestro"
+Check "maestro test subcommand gives usage instructions when called with --help" `
+    "maestro test --help" "includes" "Usage: maestro test"
+Check "maestro bugreport gives instruction" `
+    "maestro bugreport" "includes" "https://github.com/mobile-dev-inc/Maestro/issues"
+
+Write-Host ""
+Write-Host "$pass passed, $fail failed"
+if ($fail -gt 0) { exit 1 }
+exit 0

e2e/cli/test-cli.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+export MAESTRO_CLI_NO_ANALYTICS=1
+PASS=0; FAIL=0
+
+trap 'echo "ERROR: script failed at line $LINENO" >&2' ERR
+
+check() {          # check <desc> <cmd> <assertion> <expected>
+  local desc="$1" cmd="$2" assertion="$3" expected="$4"
+  local actual actual_plain ok=0
+  actual=$(eval "$cmd" 2>&1 | tr -d '\r') || true
+  actual_plain=$(printf '%s' "$actual" | sed 's/\x1b\[[0-9;]*[mGKHF]//g') # strip ansi
+  case "$assertion" in
+    equals)   [ "$actual_plain" = "$expected" ]                        && ok=1 ;;
+    includes) printf '%s' "$actual_plain" | grep -qF "$expected"       && ok=1 ;;
+    excludes) ! printf '%s' "$actual_plain" | grep -qF "$expected"     && ok=1 ;;
+    *)        echo "ERROR: unknown assertion '$assertion' (use: equals, includes, excludes)" >&2; exit 1 ;;
+  esac
+  if [ $ok -eq 1 ]; then
+    echo "PASS: $desc"
+    (( ++PASS ))
+  else
+    printf "FAIL: %s\n"        "$desc"
+    printf "      %-11s %s\n" "cmd:"       "$cmd"
+    printf "      %-11s %s\n" "assertion:" "$assertion"
+    printf "      %-11s %s\n" "expected:"  "$expected"
+    printf "      %-11s %s\n" "got:"       "$actual_plain"
+    (( ++FAIL ))
+  fi
+}
+
+VERSION=$(grep '^CLI_VERSION=' maestro-cli/gradle.properties | cut -d= -f2)
+
+# TESTS START HERE:
+
+check "maestro -v equals gradle.properties version" \
+  "maestro -v" equals "$VERSION"
+
+check "maestro gives usage instructions when called without parameters" \
+  "maestro" includes "Usage: maestro"
+
+check "maestro gives usage instructions when called with --help" \
+  "maestro --help" includes "Usage: maestro"
+
+check "maestro test subcommand gives usage instructions when called with --help" \
+  "maestro test --help" includes "Usage: maestro test"
+
+check "maestro bugreport gives instruction" \
+  "maestro bugreport" includes "https://github.com/mobile-dev-inc/Maestro/issues"
+
+echo ""
+echo "$PASS passed, $FAIL failed"
+[ $FAIL -eq 0 ]

maestro-android/build.gradle.kts

@@ -181,7 +181,7 @@ tasks.register("updateMaestroAndroidSourceSentinel") {
             .forEach { f ->
                 md.update(f.relativeTo(projectDir).invariantSeparatorsPath.toByteArray())
                 md.update(0)
-                md.update(f.readBytes())
+                md.update(f.readText(Charsets.UTF_8).replace("\r\n", "\n").toByteArray(Charsets.UTF_8)) // Keep consistent line endings for Windows builds
                 md.update(0)
             }
         val bytes = md.digest()

maestro-client/build.gradle.kts

@@ -81,7 +81,7 @@ val checkAndroidApksFresh = tasks.register("checkAndroidApksFresh") {
             .forEach { f ->
                 md.update(f.relativeTo(maestroAndroidProjectDir).invariantSeparatorsPath.toByteArray())
                 md.update(0)
-                md.update(f.readBytes())
+                md.update(f.readText(Charsets.UTF_8).replace("\r\n", "\n").toByteArray(Charsets.UTF_8)) // Keep consistent line endings for Windows builds
                 md.update(0)
             }
         val bytes = md.digest()

maestro-client/src/main/resources/maestro-android-source.sha256

@@ -1 +1 @@
-a55395029a33bab56bab234ec6d9357716fbc8a3d6202d9bc186a39255dec9d0
+34bc39fa9a97aa37556e948a67d189a76d8c01d2919729dd3d35b30946758653

maestro-client/src/main/resources/maestro-app.apk

@@ -78,3 +78,7 @@ tasks.register<Exec>("buildIosDriver") {
 tasks.named("processResources") {
     dependsOn("buildIosDriver")
 }
+
+tasks.matching { it.name == "sourcesJar" }.configureEach {
+    dependsOn("buildIosDriver")
+}