e2e.yml

name: E2E Tests

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CARGO_TERM_COLOR: always

jobs:
  changes:
    runs-on: ubuntu-latest
    timeout-minutes: 90
    outputs:
      code: ${{ steps.detect.outputs.code || 'true' }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - if: github.event_name == 'pull_request'
        id: detect
        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        with:
          predicate-quantifier: 'every'
          filters: |
            code:
              - '**'
              - '!**/*.md'
              - '!**/*.txt'
              - '!**/*.html'
              - '!docs/**'
              - '!website/**'
              - '!LICENSE'
              - '!.gitignore'

  e2e-build:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 90
    outputs:
      matrix: ${{ steps.matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Build E2E partition tool
        run: cargo build -q -p fakecloud-e2e --bin e2e_nextest_partitions

      - id: matrix
        name: Generate E2E matrix
        run: echo "matrix=$(./target/debug/e2e_nextest_partitions matrix)" >> "$GITHUB_OUTPUT"

      # Build the E2E test binaries ONCE and archive them. Every partition job
      # then runs the prebuilt binaries via `--archive-file` instead of
      # recompiling the (large) test crate from scratch — previously ~11.5 min of
      # redundant compilation paid in each of the ~19 partition jobs.
      #
      # CARGO_PROFILE_TEST_DEBUG=0 strips debuginfo from the archived binaries:
      # it shrinks both this build and the archive, so the compile, the upload
      # here, and the download in every partition are all faster (all on the
      # workflow's critical path). Assertion-failure locations are preserved via
      # std's #[track_caller], so CI failure output still points at the test line.
      - name: Archive E2E test binaries
        env:
          CARGO_PROFILE_TEST_DEBUG: "0"
        run: cargo nextest archive -P ci -p fakecloud-e2e --archive-file e2e-archive.tar.zst

      - name: Upload E2E test archive
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: fakecloud-e2e-archive
          path: e2e-archive.tar.zst
          if-no-files-found: error
          retention-days: 1

  # Build the fakecloud server binary in parallel with the test-archive build so
  # the partition jobs wait on max(archive, server) rather than their sum. The
  # harness boots this binary via its workspace-relative path
  # (crates/fakecloud-testkit/src/lib.rs), which resolves identically on every
  # GitHub-hosted runner, so a downloaded binary works regardless of where the
  # archived test binaries extract.
  e2e-fakecloud:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Build fakecloud
        run: cargo build --bin fakecloud

      - name: Upload fakecloud binary
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: fakecloud-binary-e2e
          path: target/debug/fakecloud
          if-no-files-found: error
          retention-days: 1

  e2e-partition-check:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Build E2E partition tool
        run: cargo build -q -p fakecloud-e2e --bin e2e_nextest_partitions

      - name: Verify E2E partition coverage
        run: ./target/debug/e2e_nextest_partitions check

  e2e:
    name: E2E ${{ matrix.name }}
    needs: [changes, e2e-build, e2e-fakecloud]
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.e2e-build.outputs.matrix) }}
    env:
      NEXTEST_FILTER: ${{ matrix.filter }}
      NEXTEST_PARTITION: ${{ matrix.partition }}
      INSTALL_PODMAN: ${{ matrix.install_podman }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      # No rust-cache here: this job runs prebuilt test binaries from the archive
      # and never compiles. Disk provisioning is handled by the step below, which
      # adapts to whichever disk layout the assigned runner actually has.
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest

      - name: Install podman
        if: matrix.install_podman
        run: sudo apt-get update && sudo apt-get install -y podman

      # Provision disk for BOTH the container data-root AND the nextest archive
      # extraction, adapting to the runner's actual disk layout. Two consumers
      # compete for space here: (1) container images a partition pulls (e.g.
      # lambda-runtimes-python pulls python:3.9 .. python:3.14), and (2) nextest
      # extracting the E2E test archive (every test binary, grows with each new
      # service crate) to its temp dir.
      #
      # MOST GitHub-hosted Ubuntu runners expose a ~72G root `/` with only ~16G
      # free (the image already uses ~56G) plus a SEPARATE ~74G `/mnt`
      # (/dev/sdb1, ~66G free). On those we point docker's data-root AND nextest's
      # TMPDIR at /mnt. But SOME runners have no separate /mnt (it's just a dir on
      # the root fs); there, sending both consumers to "/mnt" leaves them on the
      # 14-16G root and the archive extract dies with `No space left on device
      # (os error 28)` — which is exactly the flake #2200/#2205 chased on the
      # wrong assumption that /mnt is always separate. So: detect a real, roomy
      # separate /mnt; if absent, reclaim the root fs (drop preinstalled SDKs)
      # and keep both consumers on the freed root. Export the chosen temp base as
      # NEXTEST_TMPDIR for the run step.
      - name: Provision disk for containers + test archive
        run: |
          mnt_src=$(df --output=source /mnt 2>/dev/null | tail -1)
          root_src=$(df --output=source / | tail -1)
          mnt_avail_g=$(df -BG --output=avail /mnt 2>/dev/null | tail -1 | tr -dc '0-9')
          if [ -n "$mnt_src" ] && [ "$mnt_src" != "$root_src" ] && [ "${mnt_avail_g:-0}" -ge 40 ]; then
            tmpbase=/mnt
            echo "Separate /mnt detected ($mnt_src, ${mnt_avail_g}G free) — relocating docker + archive there"
            sudo systemctl stop docker docker.socket || true
            sudo mkdir -p /mnt/docker
            printf '{ "data-root": "/mnt/docker" }\n' | sudo tee /etc/docker/daemon.json
            sudo rm -rf /var/lib/docker
            sudo systemctl start docker
            if [ "$INSTALL_PODMAN" = "true" ]; then
              sudo mkdir -p /mnt/containers /etc/containers
              printf '[storage]\ndriver = "overlay"\ngraphroot = "/mnt/containers"\n' | sudo tee /etc/containers/storage.conf
            fi
          else
            tmpbase=/tmp
            echo "No separate /mnt (mnt_src=$mnt_src root_src=$root_src avail=${mnt_avail_g:-0}G) — reclaiming root fs instead"
            sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
              /opt/hostedtoolcache/CodeQL /usr/local/share/boost \
              "${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache}" 2>/dev/null || true
            sudo docker image prune -af || true
          fi
          sudo mkdir -p "$tmpbase/nextest-tmp"
          sudo chmod 1777 "$tmpbase/nextest-tmp"
          echo "NEXTEST_TMPDIR=$tmpbase/nextest-tmp" >> "$GITHUB_ENV"
          echo "=== df -h after provision ==="
          df -h

      - name: Prune container state before tests
        run: |
          docker system prune -af --volumes || true
          if [ "$INSTALL_PODMAN" = "true" ]; then
            podman system prune -af || true
          fi

      - name: Download fakecloud binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fakecloud-binary-e2e
          path: target/debug

      - name: Mark fakecloud binary executable
        run: chmod +x target/debug/fakecloud

      - name: Download E2E test archive
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fakecloud-e2e-archive

      - name: Disk diagnostics before nextest
        run: |
          echo "=== df -h ==="
          df -h
          echo "=== target dir ==="
          du -sh target 2>/dev/null || true
          echo "=== ~/.cargo ==="
          du -sh ~/.cargo 2>/dev/null || true

      # Run the prebuilt test binaries straight from the archive — no compilation.
      # --workspace-remap . points runtime workspace lookups at this checkout (the
      # path matches the build runner on GitHub-hosted runners).
      #
      # nextest extracts the archive to a temp dir under $TMPDIR. The provision
      # step above chose a temp base with room (separate /mnt when present, else
      # the reclaimed root) and exported it as NEXTEST_TMPDIR; point TMPDIR there
      # so the extract doesn't ENOSPC unpacking target/debug/deps/* on the small
      # root fs.
      - name: Run nextest partition
        env:
          TMPDIR: ${{ env.NEXTEST_TMPDIR }}
        run: |
          sudo mkdir -p /mnt/nextest-tmp
          sudo chmod 1777 /mnt/nextest-tmp
          cmd=(cargo nextest run -P ci
            --archive-file e2e-archive.tar.zst
            --workspace-remap .
            -E "$NEXTEST_FILTER")
          if [ -n "$NEXTEST_PARTITION" ]; then
            cmd+=(--partition "$NEXTEST_PARTITION")
          fi
          "${cmd[@]}"

      - name: Disk diagnostics after nextest
        if: always()
        run: |
          echo "=== df -h ==="
          df -h
          echo "=== target dir ==="
          du -sh target 2>/dev/null || true
          echo "=== ~/.cargo ==="
          du -sh ~/.cargo 2>/dev/null || true

      - name: Prune container state after tests
        if: always()
        run: |
          docker system prune -af --volumes || true
          if [ "$INSTALL_PODMAN" = "true" ]; then
            podman system prune -af || true
          fi

  # Privileged job: the ONLY place EC2 security-group packet filtering can be
  # exercised for real (needs nftables + CAP_NET_ADMIN on a native-Linux Docker
  # host). Boots two instances and asserts a real dropped/allowed packet —
  # closing the bug-hunt 2026-06-18 gap (a) where only the generated ruleset was
  # tested, never an actual blocked connection.
  sg-enforcement:
    name: EC2 SG enforcement (privileged)
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Install nftables
        run: sudo apt-get update && sudo apt-get install -y nftables conntrack

      # Same-subnet instances share one Linux bridge, so their traffic is L2-switched
      # and only traverses the nft `forward` chain when bridge netfilter is active.
      # The runtime tries to enable this in-process, but loading the module + setting
      # the sysctl deterministically at the host level *before* any container network
      # is created removes the race that intermittently let the "deny" packet through.
      #
      # Crucially, this is *verified*, not fire-and-forget: a silent `tee` failure (the
      # sysctl path doesn't exist until br_netfilter is loaded, or the value doesn't
      # stick) used to surface downstream as a misleading "must DROP" assertion failure
      # — the rule was installed but bridged traffic bypassed the forward chain. Read
      # the value back and fail *this* step with the real signal if it isn't 1.
      - name: Enable bridge netfilter
        run: |
          sudo modprobe br_netfilter
          # Load conntrack for the bridge family so fakecloud's bridge-family
          # SG table (`ct state established,related accept`) applies — this is
          # what makes same-subnet L2-switched traffic enforce reliably even
          # when bridge-nf-call-iptables doesn't route bridged frames to the
          # inet forward hook on this runner's kernel.
          sudo modprobe nf_conntrack_bridge || echo "nf_conntrack_bridge unavailable; bridge-family SG enforcement will degrade"
          for knob in bridge-nf-call-iptables bridge-nf-call-ip6tables bridge-nf-call-arptables; do
            echo 1 | sudo tee "/proc/sys/net/bridge/$knob" >/dev/null
          done
          got=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables)
          if [ "$got" != "1" ]; then
            echo "::error::bridge-nf-call-iptables is '$got', not 1 — br_netfilter did not engage; SG enforcement cannot filter same-subnet traffic"
            exit 1
          fi
          echo "bridge-nf-call-iptables=$got (bridge netfilter active)"

      - name: Pre-pull the instance base image
        run: docker pull alpine:3

      - name: Build fakecloud + the enforcement test
        run: |
          cargo build --bin fakecloud
          cargo test -p fakecloud-e2e --test ec2_sg_enforcement_real --no-run

      # Run as root so the spawned fakecloud process holds CAP_NET_ADMIN and can
      # install nft rules. FAKECLOUD_TEST_SG_ENFORCE=1 makes the test hard-fail
      # (not skip) if the host can't enforce. CI=1 keeps other gates strict.
      - name: Run real packet-filtering test
        run: |
          # Keep cargo's PATH but append the sbin dirs so the fakecloud process
          # (running as root) can actually find `nft` (in /usr/sbin); without
          # this its capability probe fails and enforcement silently disables.
          #
          # The scenario boots fresh instances, networks and nft state each run, so a
          # single environmental setup-race (bridge-netfilter / reconcile timing) is an
          # independent miss. Retry the whole scenario up to 3 times - every attempt
          # still asserts the real deny->allow packet transition in full; only a
          # genuine, repeatable failure fails the job.
          #
          # Each attempt is made genuinely fresh at the *host* level, not just the
          # instance level: residual nft/conntrack state is flushed AND bridge
          # netfilter is re-asserted+verified before the run. Docker can reset
          # bridge-nf-call-iptables when it (re)creates the per-subnet bridges mid-run,
          # which would otherwise poison every subsequent attempt identically — making
          # an environmental miss masquerade as a real, repeatable regression. Pinning
          # it per attempt means a failure that survives all 3 is a real enforcement
          # bug, not a leaked host-firewall state from a prior attempt.
          ensure_bridge_nf() {
            sudo modprobe br_netfilter || true
            sudo modprobe nf_conntrack_bridge || true
            echo 1 | sudo tee /proc/sys/net/bridge/bridge-nf-call-iptables >/dev/null || true
            got=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo missing)
            echo "bridge-nf-call-iptables=${got}"
          }
          # Remove leftover instance containers + per-subnet networks from any
          # prior attempt. A failed attempt panics *before* the test's own
          # cleanup runs, so its container and `fakecloud-subnet-<id>` network
          # leak. Across the 3 retries these accumulate and exhaust Docker's
          # default address pool; the next `docker network create` then fails and
          # the runtime falls back to the default bridge (docker0, 172.17.x) —
          # where same-subnet traffic never reaches fakecloud's per-subnet nft
          # rules, so the deny "bypasses" and every subsequent attempt fails
          # identically. Pruning before each attempt makes the retries genuinely
          # independent (the real fix for this job's intermittent failures).
          # `docker network rm`/`prune` alone does NOT always reclaim Docker's
          # default address pool once it gets wedged: a per-subnet bridge from a
          # prior attempt can leave the pool in a state where the next
          # `docker network create fakecloud-subnet-<id>` fails, and the runtime
          # then falls back to docker0 (172.17.x). On docker0 same-subnet traffic
          # never reaches fakecloud's per-subnet nft forward chain, so the deny
          # silently bypasses and every later attempt fails identically on a
          # 172.17.x instance IP (observed: attempt 1 on 172.18.x, attempts 2-3
          # on 172.17.0.3). A full daemon restart is the only reliable way to
          # recreate docker0 fresh and reclaim the pool; images (alpine:3) persist
          # across the restart, so this is cheap. ensure_bridge_nf runs *after*
          # this (docker recreating docker0 can reset bridge-nf-call-iptables).
          clean_docker() {
            sudo docker ps -aq --filter "label=fakecloud-ec2" | xargs -r sudo docker rm -f >/dev/null 2>&1 || true
            sudo docker network ls -q --filter "name=fakecloud-subnet-" | xargs -r sudo docker network rm >/dev/null 2>&1 || true
            sudo docker network prune -f >/dev/null 2>&1 || true
            sudo systemctl restart docker 2>/dev/null || sudo service docker restart 2>/dev/null || true
            for _ in $(seq 1 30); do sudo docker info >/dev/null 2>&1 && break; sleep 1; done
            sudo docker network prune -f >/dev/null 2>&1 || true
          }
          run_once() {
            sudo -E env "PATH=$PATH:/usr/sbin:/sbin" "HOME=$HOME" \
              FAKECLOUD_TEST_SG_ENFORCE=1 CI=1 \
              cargo test -p fakecloud-e2e --test ec2_sg_enforcement_real -- --nocapture --test-threads=1
          }
          for attempt in 1 2 3; do
            echo "=== SG enforcement attempt ${attempt}/3 ==="
            clean_docker
            ensure_bridge_nf
            if run_once; then
              echo "SG enforcement passed on attempt ${attempt}"
              exit 0
            fi
            echo "attempt ${attempt} failed; host state at failure:"
            echo "--- bridge-nf-call-iptables ---"; cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo missing
            echo "--- nft fakecloud_ec2 table ---"; sudo nft list table inet fakecloud_ec2 2>/dev/null || echo "(no table)"
            echo "--- docker networks (watch for pool exhaustion / docker0 fallback) ---"; sudo docker network ls || true
            echo "flushing nft + conntrack + leaked docker state before retry"
            sudo nft flush ruleset || true
            sudo conntrack -F || true
            sleep 3
          done
          echo "SG enforcement failed all 3 independent attempts (clean docker + bridge-nf each attempt -> real enforcement regression, not flake)"
          exit 1

      - name: nft ruleset diagnostics
        if: always()
        run: sudo nft list ruleset || true

  # Dedicated, resourced job for the Amazon MQ broker data-plane E2E. These
  # tests spawn REAL broker containers (a JVM ActiveMQ and an Erlang RabbitMQ),
  # which are far heavier than the images in the shared partition matrix, so they
  # are gated behind FAKECLOUD_E2E_MQ_BROKER=1 (skipped loudly everywhere else)
  # and run here with docker, a generous timeout for the slow broker boot, and a
  # test-step retry that absorbs genuine cold-pull/first-connection flake WITHOUT
  # masking a real regression (a persistently failing test still fails the job).
  mq-broker:
    name: MQ broker data plane
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    # Broker boot is slow (JVM/Erlang start + first cold image pull); allow ample
    # headroom over the tests' own 300s per-broker readiness windows plus retries.
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Prune container state before tests
        run: docker system prune -af --volumes || true

      # Pre-pull the broker images (with retry) so the first CreateBroker doesn't
      # pay a cold pull inside the readiness window. Retries absorb a flaky
      # registry; a persistent pull failure is surfaced (the tests would fail
      # anyway). The image refs match the runtime defaults; keep them in sync with
      # MqEngine::image (FAKECLOUD_MQ_ACTIVEMQ_IMAGE / _RABBITMQ_IMAGE).
      - name: Pre-pull broker images
        run: |
          for img in apache/activemq-classic rabbitmq:3.13-alpine; do
            for i in 1 2 3; do
              if docker pull "$img"; then break; fi
              echo "docker pull $img attempt $i failed; retrying"; sleep $((i * 5))
              [ "$i" = 3 ] && { echo "::error::could not pull $img"; exit 1; }
            done
          done

      # The testkit boots this binary via its workspace-relative path, so a plain
      # debug build in place is all the tests need.
      - name: Build fakecloud
        run: cargo build --bin fakecloud

      # Run ONLY the two broker E2E tests: the data-plane round-trips
      # (mq_dataplane: ActiveMQ STOMP + RabbitMQ AMQP) and the CFN provisioner
      # (cloudformation_mq: cfn_provisions_mq_broker_and_configuration). FAKECLOUD_
      # E2E_MQ_BROKER=1 flips them from skip to run; CI=1 makes a missing docker a
      # hard failure rather than a silent skip. --retries 2 reruns only a failed
      # test (up to 2 extra times) to ride out cold-pull / first-connection flake;
      # a test that fails every attempt still fails the job red.
      - name: Run MQ broker E2E
        env:
          FAKECLOUD_E2E_MQ_BROKER: "1"
          CI: "1"
        run: |
          cargo nextest run -P ci -p fakecloud-e2e --retries 2 \
            -E 'test(activemq_broker_delivers_a_message_over_stomp) + test(rabbitmq_broker_speaks_amqp) + test(cfn_provisions_mq_broker_and_configuration)'

      - name: Broker container diagnostics
        if: always()
        run: |
          echo "=== docker ps -a ==="
          docker ps -a || true

      - name: Prune container state after tests
        if: always()
        run: docker system prune -af --volumes || true

  # Dedicated, resourced job for the Amazon MSK (Kafka) broker data-plane E2E.
  # This test spawns a REAL single-node Apache Kafka container (a JVM broker in
  # KRaft mode), far heavier than the images in the shared partition matrix, so
  # it is gated behind FAKECLOUD_E2E_MSK=1 (skipped loudly everywhere else) and
  # run here with docker, a generous timeout for the slow broker boot, and a
  # test-step retry that absorbs genuine cold-pull/first-connection flake WITHOUT
  # masking a real regression (a persistently failing test still fails the job).
  # The client is pure-Rust (rskafka), so there is no native/cmake build step.
  msk-broker:
    name: MSK Kafka data plane
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    # Broker boot is slow (JVM/KRaft start + first cold image pull); allow ample
    # headroom over the test's own 300s cluster-readiness window plus retries.
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Prune container state before tests
        run: docker system prune -af --volumes || true

      # Pre-pull the Kafka image (with retry) so the first CreateCluster doesn't
      # pay a cold pull inside the readiness window. Retries absorb a flaky
      # registry; a persistent pull failure is surfaced (the test would fail
      # anyway). The image ref matches the runtime default; keep it in sync with
      # KafkaRuntime::image (FAKECLOUD_KAFKA_IMAGE).
      - name: Pre-pull Kafka image
        run: |
          for i in 1 2 3; do
            if docker pull apache/kafka:3.8.0; then break; fi
            echo "docker pull apache/kafka:3.8.0 attempt $i failed; retrying"; sleep $((i * 5))
            [ "$i" = 3 ] && { echo "::error::could not pull apache/kafka:3.8.0"; exit 1; }
          done

      # The testkit boots this binary via its workspace-relative path, so a plain
      # debug build in place is all the test needs.
      - name: Build fakecloud
        run: cargo build --bin fakecloud

      # Run ONLY the MSK data-plane round-trip test. FAKECLOUD_E2E_MSK=1 flips it
      # from skip to run; CI=1 makes a missing docker a hard failure rather than a
      # silent skip. --retries 2 reruns only a failed test (up to 2 extra times)
      # to ride out cold-pull / first-connection flake; a test that fails every
      # attempt still fails the job red.
      - name: Run MSK broker E2E
        env:
          FAKECLOUD_E2E_MSK: "1"
          CI: "1"
        run: |
          cargo nextest run -P ci -p fakecloud-e2e --retries 2 \
            -E 'test(msk_cluster_delivers_a_message_through_a_real_kafka_broker) + test(cfn_provisions_msk_broker_cluster)'

      - name: Broker container diagnostics
        if: always()
        run: |
          echo "=== docker ps -a ==="
          docker ps -a || true

      - name: Prune container state after tests
        if: always()
        run: docker system prune -af --volumes || true

  flink-runtime:
    name: Managed Flink data plane
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    # A Flink session cluster boot is slow (JVM JobManager + TaskManager start +
    # first cold image pull + job submission); allow ample headroom over the
    # test's own 360s app-readiness window plus retries.
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: ./.github/actions/free-disk
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
      - uses: taiki-e/install-action@b3bd89dcd46d5f3d508436e1bf794284c155bbcc # nextest
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

      - name: Prune container state before tests
        run: docker system prune -af --volumes || true

      # Pre-pull the Flink image (with retry) so the first StartApplication
      # doesn't pay a cold pull inside the readiness window. The image ref
      # matches the runtime default; keep it in sync with FlinkRuntime::image
      # (FAKECLOUD_KINESISANALYTICSV2_IMAGE).
      - name: Pre-pull Flink image
        run: |
          for i in 1 2 3; do
            if docker pull flink:1.19; then break; fi
            echo "docker pull flink:1.19 attempt $i failed; retrying"; sleep $((i * 5))
            [ "$i" = 3 ] && { echo "::error::could not pull flink:1.19"; exit 1; }
          done

      # The testkit boots this binary via its workspace-relative path, so a plain
      # debug build in place is all the test needs.
      - name: Build fakecloud
        run: cargo build --bin fakecloud

      # Run ONLY the Flink data-plane round-trip test. FAKECLOUD_E2E_FLINK=1 flips
      # it from skip to run; CI=1 makes a missing docker a hard failure rather
      # than a silent skip. --retries 2 reruns only a failed test to ride out
      # cold-pull / first-connection flake; a test that fails every attempt still
      # fails the job red.
      - name: Run Flink runtime E2E
        env:
          FAKECLOUD_E2E_FLINK: "1"
          CI: "1"
        run: |
          cargo nextest run -P ci -p fakecloud-e2e --retries 2 \
            -E 'test(flink_application_runs_a_real_job_in_a_container)'

      - name: Flink container diagnostics
        if: always()
        run: |
          echo "=== docker ps -a ==="
          docker ps -a || true

      - name: Prune container state after tests
        if: always()
        run: docker system prune -af --volumes || true