feat(nimbus): add Fenix end-to-end enrollment integration test

.github/workflows/fenix-integration-test.yml

@@ -0,0 +1,120 @@
+name: Fenix Enrollment Integration Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "3 12 * * *"
+  push:
+    branches:
+      - main
+      - update_firefox_fenix_beta
+      - update_firefox_fenix_release
+  pull_request:
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  test:
+    name: "Fenix Enrollment (${{ matrix.channel }})"
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - channel: beta
+            build_env: experimenter/tests/firefox_fenix_beta_build.env
+            task_id_var: FIREFOX_FENIX_BETA_TASK_ID
+          - channel: release
+            build_env: experimenter/tests/firefox_fenix_release_build.env
+            task_id_var: FIREFOX_FENIX_RELEASE_TASK_ID
+    env:
+      INTEGRATION_TEST_NGINX_URL: https://localhost
+      FENIX_APK_PATH: ${{ github.workspace }}/fenix.apk
+      FENIX_CHANNEL: ${{ matrix.channel }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/check-changed-paths
+        id: check-paths
+        with:
+          paths: "experimenter/experimenter/ experimenter/tests/integration/ experimenter/tests/firefox_fenix_beta_build.env experimenter/tests/firefox_fenix_release_build.env"
+
+      - uses: ./.github/actions/setup-cached-build
+        if: steps.check-paths.outputs.should-run == 'true'
+
+      - name: Download Fenix ${{ matrix.channel }} APK
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: |
+          . ${{ matrix.build_env }}
+          if [ -z "${!TASK_ID_VAR}" ]; then
+            echo "::error::$TASK_ID_VAR is empty in ${{ matrix.build_env }}. Run update-firefox.yml with the ${{ matrix.channel == 'beta' && 'fenix-beta' || 'fenix-release' }} variant to populate it."
+            exit 1
+          fi
+          curl -sSfL -o "$FENIX_APK_PATH" \
+            "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/${!TASK_ID_VAR}/artifacts/public/build/target.x86_64.apk"
+        env:
+          TASK_ID_VAR: ${{ matrix.task_id_var }}
+
+      - name: Install nimbus-cli
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/mozilla/application-services/main/install-nimbus-cli.sh -o /tmp/install-nimbus-cli.sh
+          sudo bash /tmp/install-nimbus-cli.sh --directory /usr/local/bin
+
+      - name: Set up Python
+        if: steps.check-paths.outputs.should-run == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install poetry
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: pipx install poetry
+
+      - name: Bring up Experimenter stack
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: |
+          cp .env.integration-tests .env
+          make refresh SKIP_DUMMY=1 up_prod_detached
+
+      - name: Wait for Experimenter backend to be ready
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: curl --retry 60 --retry-delay 5 --retry-all-errors -sfk -o /dev/null https://localhost/__lbheartbeat__
+
+      - name: Enable KVM
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Prepare Android SDK install dir
+        if: steps.check-paths.outputs.should-run == 'true'
+        run: |
+          sudo mkdir -p /usr/local/lib/android
+          sudo chown -R "$USER" /usr/local/lib/android
+
+      - name: Run Fenix enrollment test on emulator
+        if: steps.check-paths.outputs.should-run == 'true'
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 34
+          arch: x86_64
+          target: google_apis
+          profile: pixel_6
+          disable-animations: true
+          disk-size: 4096M
+          emulator-options: -no-window -no-boot-anim -no-audio -accel on -gpu swiftshader_indirect
+          script: make integration_test_nimbus_fenix
+
+      - name: Upload test report on failure
+        if: failure() && steps.check-paths.outputs.should-run == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: fenix-${{ matrix.channel }}-test-report
+          path: experimenter/tests/integration/test-reports/
+          if-no-files-found: ignore

.github/workflows/update-firefox.yml

@@ -13,6 +13,8 @@ on:
           - all
           - desktop-release
           - desktop-beta
+          - fenix-beta
+          - fenix-release
 
 jobs:
   update:
@@ -31,6 +33,12 @@ jobs:
           - application: desktop
             channel: beta
             display_name: Firefox Desktop Beta
+          - application: fenix
+            channel: beta
+            display_name: Firefox Fenix Beta
+          - application: fenix
+            channel: release
+            display_name: Firefox Fenix Release
     steps:
       - name: Check if this variant should run
         id: should-run

Makefile

@@ -258,8 +258,9 @@ integration_test_nimbus_sdk: build_integration_test build_prod
 	MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION_RUN) -it rust-sdk sh -c "./experimenter/tests/nimbus_rust_tests.sh"
 
 integration_test_nimbus_fenix:
-	poetry -C experimenter/tests/integration/ -vvv install --no-root
-	poetry -C experimenter/tests/integration/ -vvv run pytest --html=workspace/test-results/report.htm --self-contained-html --reruns-delay 30 --driver Firefox experimenter/tests/integration/nimbus/android --junitxml=experimenter/tests/integration/test-reports/experimenter_fenix_integration_tests.xml -vvv
+	poetry -C experimenter/tests install --no-root
+	mkdir -p experimenter/tests/integration/test-reports
+	cd experimenter/tests && poetry run pytest -m fenix_enrollment -o addopts= -p no:rerunfailures --junitxml=integration/test-reports/fenix_enrollment.xml integration/nimbus/android $(PYTEST_ARGS)
 
 # cirrus
 CIRRUS_ENABLE = export CIRRUS=1 &&

experimenter/tests/firefox_fenix_release_build.env

@@ -1 +1 @@
-FIREFOX_FENIX_RELEASE_TASK_ID="YmbxEtM6QqGQLzrwS532aw"
+FIREFOX_FENIX_RELEASE_TASK_ID="aYsyO389TuSLoLVOKpvGsg"

experimenter/tests/integration/nimbus/android/conftest.py

@@ -0,0 +1,69 @@
+import os
+import time
+
+import pytest
+import requests
+
+from nimbus.models.base_dataclass import BaseExperimentApplications
+from nimbus.utils import helpers
+
+FENIX_APP = BaseExperimentApplications.FIREFOX_FENIX.value
+RECIPE_POLL_TIMEOUT = 60
+
+
+@pytest.fixture(name="fenix_channel")
+def fixture_fenix_channel():
+    return os.environ["FENIX_CHANNEL"]
+
+
+@pytest.fixture(name="fenix_apk_path")
+def fixture_fenix_apk_path():
+    return os.environ["FENIX_APK_PATH"]
+
+
+@pytest.fixture
+def experiment_slug(fenix_channel):
+    return f"fenix-{fenix_channel}-integration-test"
+
+
+def wait_for_recipe(slug):
+    base_url = os.environ.get("INTEGRATION_TEST_NGINX_URL", "https://nginx")
+    url = f"{base_url}/api/v6/experiments/{slug}/"
+    deadline = time.time() + RECIPE_POLL_TIMEOUT
+    last_error = None
+    while time.time() < deadline:
+        try:
+            resp = requests.get(url, verify=False, timeout=5)
+            if resp.status_code == 200:
+                recipe = resp.json()
+                if recipe.get("slug") == slug and recipe.get("bucketConfig"):
+                    return recipe
+                last_error = (
+                    f"recipe missing bucketConfig: {recipe.get('bucketConfig')!r}"
+                )
+            else:
+                last_error = f"HTTP {resp.status_code}"
+        except (requests.RequestException, ValueError) as exc:
+            last_error = str(exc)
+        time.sleep(1)
+    pytest.fail(f"Timed out waiting for recipe at {url} ({last_error})")
+
+
+@pytest.fixture
+def create_fenix_experiment(application_feature_ids):
+    def _create_fenix_experiment(slug, channel):
+        feature_id = application_feature_ids[FENIX_APP]
+        helpers.create_experiment(
+            slug,
+            FENIX_APP,
+            data={
+                "feature_config_ids": [int(feature_id)],
+                "channel": channel,
+                "firefox_min_version": "",
+            },
+            targeting="no_targeting",
+        )
+        helpers.launch_to_preview(slug)
+        return wait_for_recipe(slug)
+
+    return _create_fenix_experiment

experimenter/tests/integration/nimbus/android/test_fenix_enrollment.py

@@ -0,0 +1,77 @@
+import json
+import re
+import subprocess
+import time
+
+import pytest
+
+from nimbus.models.base_dataclass import BaseExperimentApplications
+
+FENIX_APP = BaseExperimentApplications.FIREFOX_FENIX.value
+APP_APPLY_WAIT = 15
+LOG_STATE_WAIT = 5
+
+
+@pytest.mark.fenix_enrollment
+def test_fenix_enrollment(
+    fenix_channel,
+    fenix_apk_path,
+    experiment_slug,
+    create_fenix_experiment,
+    tmp_path,
+):
+    recipe = create_fenix_experiment(experiment_slug, fenix_channel)
+
+    recipe_path = tmp_path / "fenix_recipe.json"
+    recipe_path.write_text(json.dumps(recipe))
+
+    subprocess.check_call(["adb", "install", fenix_apk_path])
+    subprocess.check_call(["adb", "logcat", "-c"])
+
+    # Prevent the real Nimbus RS fetch from overwriting our local enrollment
+    # (without this, Nimbus evolves against the fresh fetch — which does not
+    # contain our test experiment — and unenrolls us).
+    subprocess.check_call(["adb", "shell", "svc", "wifi", "disable"])
+    subprocess.check_call(["adb", "shell", "svc", "data", "disable"])
+
+    subprocess.check_call(
+        [
+            "nimbus-cli",
+            "--app",
+            FENIX_APP,
+            "--channel",
+            fenix_channel,
+            "enroll",
+            experiment_slug,
+            "--branch",
+            "control",
+            "--file",
+            str(recipe_path),
+            "--preserve-targeting",
+            "--preserve-bucketing",
+            "--reset-app",
+            "--no-validate",
+        ]
+    )
+    time.sleep(APP_APPLY_WAIT)
+
+    subprocess.check_call(
+        ["nimbus-cli", "--app", FENIX_APP, "--channel", fenix_channel, "log-state"]
+    )
+    time.sleep(LOG_STATE_WAIT)
+
+    logcat = subprocess.check_output(["adb", "logcat", "-d"], text=True)
+
+    pattern = re.compile(
+        rf"nimbus_client:\s*{re.escape(experiment_slug)}\s+\|\s*\S+\s+\|\s*(\S+)"
+    )
+    match = pattern.search(logcat)
+    nimbus_lines = [line for line in logcat.splitlines() if "nimbus_client" in line]
+    assert match is not None, (
+        f"No log-state row found for {experiment_slug}.\n"
+        f"--- last 30 nimbus_client lines ---\n" + "\n".join(nimbus_lines[-30:])
+    )
+    enrolled_branch = match.group(1)
+    assert enrolled_branch in {"control", "treatment-a"}, (
+        f"Unexpected branch {enrolled_branch!r} for {experiment_slug}"
+    )