test(integration): full Docker-based integration test suite

160+ shell tests covering: messaging, tasks, files, pub/sub, trust, networks,
policies, gateway, NAT, chaos/loss, security, observability, race conditions,
durability, and multi-topology scenarios (ring4, star5, splitbrain, mesh).
Includes docker-compose topologies, helper libraries, and a TEST-PLAN.md.

Author: teovl

tests/integration/COVERAGE-REPORT.md

@@ -0,0 +1,28 @@
+FROM golang:1.25-bookworm AS builder
+WORKDIR /workspace
+COPY . .
+RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /bin/pilot-rendezvous ./cmd/rendezvous && \
+    CGO_ENABLED=0 go build -ldflags="-s -w" -o /bin/pilot-daemon ./cmd/daemon && \
+    CGO_ENABLED=0 go build -ldflags="-s -w" -o /bin/pilotctl ./cmd/pilotctl && \
+    CGO_ENABLED=0 go build -ldflags="-s -w" -o /bin/pilot-gateway ./cmd/gateway
+
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash curl jq procps ca-certificates iproute2 python3 xxd \
+    && rm -rf /var/lib/apt/lists/*
+# Note: Chunk D security tests also need tcpdump for frame capture; they
+# install it on-demand via ensure_tcpdump() in sec_helpers.sh (apt-get
+# install -y tcpdump) so we don't bloat every agent image with it.
+COPY --from=builder /bin/pilot-rendezvous /usr/local/bin/
+COPY --from=builder /bin/pilot-daemon /usr/local/bin/
+COPY --from=builder /bin/pilotctl /usr/local/bin/
+COPY --from=builder /bin/pilot-gateway /usr/local/bin/
+RUN mkdir -p /root/.pilot/inbox /root/.pilot/received
+COPY tests/integration/test_p2p.sh /tests/
+# Security-test injector + helpers (Chunk D). Kept alongside /tests so
+# raw_udp.py can be invoked as `python3 /tests/raw_udp.py ...` from any
+# service that has python3 on PATH.
+COPY tests/integration/raw_udp.py /tests/
+COPY tests/integration/sec_helpers.sh /tests/
+RUN chmod +x /tests/test_p2p.sh /tests/raw_udp.py
+WORKDIR /tests

tests/integration/Dockerfile.multi

@@ -0,0 +1,10 @@
+FROM alpine:3.20
+RUN apk add --no-cache iptables ip6tables iproute2 bash
+# Entrypoint is supplied per-compose-service via `command:` — the gateway
+# containers set up masquerading rules keyed to a `NAT_MODE` env var:
+#   - cone       : stateful NAT (Linux default; behaves ~ port-restricted cone)
+#   - symmetric  : force per-destination source-port randomisation
+# Usage (from compose):
+#   command: ["/bin/bash", "/nat-gw.sh"]
+COPY nat-gw.sh /nat-gw.sh
+RUN chmod +x /nat-gw.sh

tests/integration/Dockerfile.nat-gw

@@ -0,0 +1,109 @@
+#!/bin/bash
+# chaos_helpers.sh — sourceable helpers that apply tc netem + iptables
+# chaos inside the compose agent containers.
+#
+# Usage:
+#   source chaos_helpers.sh
+#   DC="docker compose -f docker-compose.multi.yml -f docker-compose.multi.chaos.yml"
+#   apply_loss agent-b 30
+#   ...
+#   strip_chaos agent-b
+#
+# All helpers take the service name (agent-a / agent-b / rendezvous) as
+# the first argument and shell out to `$DC exec -T`. The overlay compose
+# file `docker-compose.multi.chaos.yml` grants NET_ADMIN; without it
+# every one of these will fail with "Operation not permitted".
+#
+# Helpers are idempotent in the sense that strip_chaos is safe to call
+# even when no qdisc is present, and iptables rules are keyed on
+# --comment pilot-chaos so repeated partition/heal don't leak state.
+#
+# DC env var MUST be set by the caller — we do not hardcode the compose
+# file list so tests can compose additional overlays (webhooks, etc.).
+
+: "${DC:?chaos_helpers: caller must export DC (docker compose -f ...)}"
+
+# Apply symmetric packet loss on a service's eth0 (root qdisc).
+# apply_loss <service> <percent>
+apply_loss() {
+    local svc="$1" pct="$2"
+    # Clean any previous root qdisc first so repeated calls work.
+    $DC exec -T "$svc" tc qdisc del dev eth0 root 2>/dev/null || true
+    $DC exec -T "$svc" tc qdisc add dev eth0 root netem loss "${pct}%"
+}
+
+# Reorder pct% of packets with a 50% correlation (default netem shape).
+# apply_reorder <service> <percent>
+apply_reorder() {
+    local svc="$1" pct="$2"
+    $DC exec -T "$svc" tc qdisc del dev eth0 root 2>/dev/null || true
+    # netem reorder requires a nonzero delay to actually reorder packets
+    # (else there is no queue to shuffle). 10ms is fine for local tests.
+    $DC exec -T "$svc" tc qdisc add dev eth0 root netem delay 10ms reorder "${pct}%" 50%
+}
+
+# Add fixed delay (ms) with +/-50ms jitter.
+# apply_delay <service> <ms>
+apply_delay() {
+    local svc="$1" ms="$2"
+    $DC exec -T "$svc" tc qdisc del dev eth0 root 2>/dev/null || true
+    $DC exec -T "$svc" tc qdisc add dev eth0 root netem delay "${ms}ms" 50ms distribution normal
+}
+
+# Strip any netem qdisc AND iptables chaos rules from a service.
+# strip_chaos <service>
+strip_chaos() {
+    local svc="$1"
+    $DC exec -T "$svc" tc qdisc del dev eth0 root 2>/dev/null || true
+    # Clear any partition rules we may have inserted.
+    # We tag all of our rules with --comment pilot-chaos for easy removal.
+    $DC exec -T "$svc" bash -c '
+        while iptables -C INPUT -m comment --comment pilot-chaos -j DROP 2>/dev/null; do
+            iptables -D INPUT -m comment --comment pilot-chaos -j DROP || break
+        done
+        while iptables -C OUTPUT -m comment --comment pilot-chaos -j DROP 2>/dev/null; do
+            iptables -D OUTPUT -m comment --comment pilot-chaos -j DROP || break
+        done
+        # Src/dst-specific partition drops (dropped by remove below too).
+        iptables -S INPUT 2>/dev/null | grep -- "--comment pilot-chaos" | while read -r line; do
+            rule=$(echo "$line" | sed "s/^-A /-D /")
+            iptables $rule 2>/dev/null || true
+        done
+        iptables -S OUTPUT 2>/dev/null | grep -- "--comment pilot-chaos" | while read -r line; do
+            rule=$(echo "$line" | sed "s/^-A /-D /")
+            iptables $rule 2>/dev/null || true
+        done
+        true
+    ' 2>/dev/null || true
+}
+
+# Partition src from dst (one-way drop, run twice for bidirectional).
+# apply_partition <src_service> <dst_ip>
+#
+# Note: we must pass an IP, not a hostname, because iptables inside the
+# container cannot resolve compose service names reliably. Callers
+# typically resolve with: $DC exec -T rendezvous getent hosts agent-b
+apply_partition() {
+    local src="$1" dst_ip="$2"
+    # Drop both directions so tunnel traffic cannot sneak out on return
+    # path. Tag with --comment pilot-chaos for strip_chaos cleanup.
+    $DC exec -T "$src" iptables -I OUTPUT -d "$dst_ip" -m comment --comment pilot-chaos -j DROP
+    $DC exec -T "$src" iptables -I INPUT -s "$dst_ip" -m comment --comment pilot-chaos -j DROP
+}
+
+# Heal a partition previously applied with apply_partition.
+# heal_partition <src_service> <dst_ip>
+heal_partition() {
+    local src="$1" dst_ip="$2"
+    $DC exec -T "$src" iptables -D OUTPUT -d "$dst_ip" -m comment --comment pilot-chaos -j DROP 2>/dev/null || true
+    $DC exec -T "$src" iptables -D INPUT -s "$dst_ip" -m comment --comment pilot-chaos -j DROP 2>/dev/null || true
+}
+
+# Resolve a compose service hostname to its network IP from INSIDE the
+# compose network. Uses rendezvous as the resolver container (always up
+# in this stack). Echoes the v4 IP or empty.
+# resolve_service_ip <service>
+resolve_service_ip() {
+    local svc="$1"
+    $DC exec -T rendezvous getent hosts "$svc" 2>/dev/null | awk '{print $1}' | head -n1
+}

tests/integration/TEST-PLAN.md

@@ -0,0 +1,76 @@
+# Gateway overlay. Stack with:
+#   docker compose -f docker-compose.multi.yml -f docker-compose.multi.gateway.yml up -d
+#
+# Adds a `gateway` service that:
+#   - runs its own pilot-daemon (so it's a full node on the overlay)
+#   - runs pilot-gateway mapping agent-b to a local IP in 10.4.0.0/16
+#   - exposes mapped ports on the docker network so tests can curl / nc them
+#
+# pilot-gateway today is a TCP bridge: it listens on a loopback alias for
+# each mapped pilot address on ports 80, 443, 1000, 1001, 1002, 7, 8080,
+# 8443 and forwards TCP streams over the pilot tunnel to <pilot-addr>:<port>.
+# It does NOT expose an HTTP/gRPC control-plane API of its own — all
+# operations (task submit, trust grant, polo read, pubsub, lookup, register,
+# rotate-key, ping) continue to flow through pilotctl / the daemon socket.
+# Tests therefore exercise "through the gateway" by driving pilotctl against
+# the gateway-side daemon, OR by hitting the mapped local IP:port with
+# standard TCP/HTTP tooling.
+#
+# NET_ADMIN is required so the gateway can add a loopback alias via `ip
+# addr add` (pkg/gateway/gateway.go addLoopbackAlias).
+services:
+  gateway:
+    build:
+      context: ../..
+      dockerfile: tests/integration/Dockerfile.multi
+    hostname: gateway
+    depends_on:
+      rendezvous:
+        condition: service_healthy
+      agent-a:
+        condition: service_started
+      agent-b:
+        condition: service_started
+    networks:
+      pilot-p2p:
+        ipv4_address: ${PILOT_SUBNET_PREFIX:-172.29.0}.40
+    environment:
+      PILOT_REGISTRY: "${PILOT_SUBNET_PREFIX:-172.29.0}.10:9000"
+      PILOT_SUBNET_PREFIX: "${PILOT_SUBNET_PREFIX:-172.29.0}"
+    # NET_ADMIN is required by pilot-gateway to `ip addr add` a loopback
+    # alias for each mapped local IP. Without it, the proxy listeners
+    # never bind successfully.
+    cap_add:
+      - NET_ADMIN
+    volumes:
+      - ./results:/results
+    # The container runs the daemon, then the gateway. The test scripts
+    # exec into this container to drive pilotctl and curl through the
+    # mapped IPs.
+    command: >
+      bash -c "
+        mkdir -p /root/.pilot/inbox /root/.pilot/received &&
+        pilot-daemon
+          -registry ${PILOT_SUBNET_PREFIX:-172.29.0}.10:9000
+          -beacon ${PILOT_SUBNET_PREFIX:-172.29.0}.10:9001
+          -hostname gateway
+          -identity /root/.pilot/identity.json
+          -email gateway@p2p.test
+          -endpoint ${PILOT_SUBNET_PREFIX:-172.29.0}.40:4000
+          -listen :4000
+          -public
+          -keepalive 5s
+          -log-level info
+          > /tmp/gateway-daemon.log 2>&1 &
+        DAEMON_PID=$$! ;
+        for i in $$(seq 1 30); do
+          if [ -S /tmp/pilot.sock ] && pilotctl --json info 2>/dev/null | jq -e '.data.node_id' >/dev/null 2>&1; then
+            break;
+          fi;
+          sleep 1;
+        done;
+        pilot-gateway -subnet 10.4.0.0/16 -socket /tmp/pilot.sock run > /tmp/gateway.log 2>&1 &
+        GW_PID=$$! ;
+        echo 'gateway ready';
+        wait $$DAEMON_PID $$GW_PID
+      "

tests/integration/chaos_helpers.sh

@@ -0,0 +1,148 @@
+# NAT overlay — places agent-a on a private bridge behind a NAT gateway,
+# while rendezvous + agent-b sit on the public bridge. Used by
+# tests/integration/test_nat_*.sh to exercise Pilot's STUN + beacon
+# hole-punch + relay fallback paths.
+#
+# Topology (public = 172.30.10.0/24, private-a = 172.30.11.0/24):
+#
+#   agent-a (172.30.11.20) --> nat-gw (11.1 priv / 10.30 pub) --> public
+#                                                                  ├── rendezvous (172.30.10.10)
+#                                                                  └── agent-b   (172.30.10.21)
+#
+# The nat-gw container runs iptables MASQUERADE (or random-fully SNAT
+# in NAT_MODE=symmetric) to translate agent-a's egress traffic. agent-a's
+# default route is rewritten to point at the gateway.
+#
+# Bring up:
+#   docker compose -f docker-compose.multi.yml \
+#                  -f docker-compose.multi.nat.yml up -d
+# NOTE: This overlay REPLACES the network config from the base compose
+# because it needs two bridges. Tests run `docker compose -f
+# docker-compose.multi.nat.yml ...` standalone (no base overlay) for
+# NAT scenarios.
+
+services:
+  rendezvous:
+    build:
+      context: ../..
+      dockerfile: tests/integration/Dockerfile.multi
+    hostname: rendezvous
+    networks:
+      public:
+        ipv4_address: 172.30.10.10
+    command: >
+      pilot-rendezvous
+      -registry-addr 0.0.0.0:9000
+      -beacon-addr 0.0.0.0:9001
+      -http 0.0.0.0:8080
+      -log-level info
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8080/api/stats >/dev/null || exit 1"]
+      interval: 2s
+      timeout: 2s
+      retries: 10
+
+  nat-gw:
+    build:
+      context: .
+      dockerfile: Dockerfile.nat-gw
+    hostname: nat-gw
+    cap_add:
+      - NET_ADMIN
+    sysctls:
+      - net.ipv4.ip_forward=1
+    environment:
+      NAT_MODE: "${NAT_MODE:-cone}"
+    # Attach private FIRST (eth0) then public (eth1). Docker names
+    # interfaces in attachment order; the nat-gw.sh script reads
+    # PUBLIC_IFACE / PRIVATE_IFACE env.
+    networks:
+      private-a:
+        ipv4_address: 172.30.11.2
+      public:
+        ipv4_address: 172.30.10.30
+    command: ["/bin/bash", "/nat-gw.sh"]
+
+  agent-a:
+    build:
+      context: ../..
+      dockerfile: tests/integration/Dockerfile.multi
+    hostname: agent-a
+    depends_on:
+      rendezvous:
+        condition: service_healthy
+      nat-gw:
+        condition: service_started
+    cap_add:
+      - NET_ADMIN
+    networks:
+      private-a:
+        ipv4_address: 172.30.11.20
+    environment:
+      PILOT_REGISTRY: "172.30.10.10:9000"
+    # Fix the default route to egress through the NAT gateway, then
+    # exec the daemon. Using array form avoids YAML folding eating
+    # flag tokens in the flow-scalar bash -c '…' path.
+    entrypoint:
+      - /bin/bash
+      - -c
+      - |
+        set -e
+        ip route del default 2>/dev/null || true
+        ip route add default via 172.30.11.2
+        echo "[agent-a] routes:"
+        ip route
+        exec pilot-daemon \
+          -registry 172.30.10.10:9000 \
+          -beacon   172.30.10.10:9001 \
+          -hostname agent-a \
+          -identity /root/.pilot/identity.json \
+          -email agent-a@nat.test \
+          -listen :4000 \
+          -keepalive 5s \
+          -log-level info
+    command: []
+
+  agent-b:
+    build:
+      context: ../..
+      dockerfile: tests/integration/Dockerfile.multi
+    hostname: agent-b
+    depends_on:
+      rendezvous:
+        condition: service_healthy
+    networks:
+      public:
+        ipv4_address: 172.30.10.21
+    environment:
+      PILOT_REGISTRY: "172.30.10.10:9000"
+    command: >
+      pilot-daemon
+      -registry 172.30.10.10:9000
+      -beacon   172.30.10.10:9001
+      -hostname agent-b
+      -identity /root/.pilot/identity.json
+      -email agent-b@nat.test
+      -endpoint 172.30.10.21:4000
+      -listen :4000
+      -public
+      -keepalive 5s
+      -log-level info
+
+networks:
+  public:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.30.10.0/24
+  private-a:
+    driver: bridge
+    # `internal: true` would forbid the bridge from NATing out via the
+    # host. We WANT that forbidden — the only egress path for agent-a
+    # is through nat-gw. But Docker's built-in NAT on a non-internal
+    # bridge would let agent-a bypass the gateway. So we mark private-a
+    # internal; agent-a reaches the public subnet only via 172.30.11.2.
+    internal: true
+    ipam:
+      config:
+        - subnet: 172.30.11.0/24