aws
diff --git a/‎.github/scripts/efa/ec2_helpers.py‎
Lines changed: 47 additions & 6 deletions b/‎.github/scripts/efa/ec2_helpers.py‎
Lines changed: 47 additions & 6 deletions
diff --git a/‎.github/workflows/pr-pytorch-ec2-cuda.yml‎
Lines changed: 20 additions & 4 deletions b/‎.github/workflows/pr-pytorch-ec2-cuda.yml‎
Lines changed: 20 additions & 4 deletions
diff --git a/‎.github/workflows/pr-vllm-ec2.yml‎
Lines changed: 36 additions & 0 deletions b/‎.github/workflows/pr-vllm-ec2.yml‎
Lines changed: 36 additions & 0 deletions
diff --git a/‎.github/workflows/reusable-efa-tests.yml‎
Lines changed: 32 additions & 2 deletions b/‎.github/workflows/reusable-efa-tests.yml‎
Lines changed: 32 additions & 2 deletions
diff --git a/‎test/efa/scripts/nccl_allreduce.sh‎
Lines changed: 48 additions & 4 deletions b/‎test/efa/scripts/nccl_allreduce.sh‎
Lines changed: 48 additions & 4 deletions
@@ -274,17 +274,37 @@ def launch_efa_instances(aws_session, ami_id, instance_type, key_name, sg_id, co
     )
 
 
-def setup_container(conn, image_uri, container_name):
-    """Pull image and start container with EFA devices and host networking."""
+def setup_container(conn, image_uri, container_name, entrypoint=None, cmd="bash"):
+    """Pull image and start container with EFA devices and host networking.
+
+    The container is launched detached (`-id`) and must stay alive long enough
+    for the test to docker-exec into it. The right way to keep it alive depends
+    on the image's entrypoint, which is image-specific — so the caller decides
+    via `entrypoint` / `cmd` instead of this helper sniffing the URI:
+
+    - `entrypoint=None` (default): use the image's baked-in entrypoint and pass
+      `cmd` as its argv. Suitable when the entrypoint sets up runtime state we
+      need (e.g., PyTorch's entrypoint.sh sets LD_LIBRARY_PATH for CUDA
+      forward-compat before `exec "$@"`, which NCCL+EFA later inherits). With
+      `cmd="bash"`, `-id` keeps stdin open and bash blocks reading from it.
+    - `entrypoint="/bin/bash"` + `cmd="-c 'sleep infinity'"`: override when the
+      baked-in entrypoint can't accept a generic shell arg (e.g., vLLM's
+      `dockerd_entrypoint.sh` execs `vllm serve "$@"` and would parse `bash`
+      as a model tag). Once entrypoint is `/bin/bash`, the CMD becomes argv
+      to bash, so an explicit `-c` payload is required — leaving CMD empty
+      would inherit the image's CMD as bash args.
+    """
     devices = get_efa_devices(conn)
     device_args = " ".join(f"--device {d}" for d in devices)
+    entrypoint_arg = f"--entrypoint {entrypoint}" if entrypoint else ""
 
     conn.run(f"docker rm -f {container_name}", warn=True)
     conn.run(
         f"docker run --runtime=nvidia --gpus all -id "
         f"--name {container_name} --network host --ulimit memlock=-1:-1 "
+        f"{entrypoint_arg} "
         f"{device_args} -v $HOME/test:/test -v /dev/shm:/dev/shm "
-        f"{image_uri} bash"
+        f"{image_uri} {cmd}"
     )
     LOGGER.info(f"Started container {container_name}")
 
@@ -387,9 +407,18 @@ def release_eip(aws_session, alloc_id):
 
 
 @contextmanager
-def efa_instances(image_uri, instance_type="p4d.24xlarge", region=DEFAULT_REGION):
+def efa_instances(
+    image_uri,
+    instance_type="p4d.24xlarge",
+    region=DEFAULT_REGION,
+    container_entrypoint=None,
+    container_cmd="bash",
+):
     """Context manager that launches 2 EFA instances, sets up containers + SSH, and cleans up.
 
+    `container_entrypoint` / `container_cmd` are forwarded to setup_container — see
+    its docstring for when overriding the entrypoint is appropriate.
+
     Yields (master_conn, worker_conn, aws_session) where connections are to the EC2 hosts.
     """
     aws_session = AWSSessionManager(region=region)
@@ -463,8 +492,20 @@ def efa_instances(image_uri, instance_type="p4d.24xlarge", region=DEFAULT_REGION
         master_conn.run(f"docker pull {image_uri}")
         worker_conn.run(f"docker pull {image_uri}")
 
-        setup_container(master_conn, image_uri, MASTER_CONTAINER_NAME)
-        setup_container(worker_conn, image_uri, WORKER_CONTAINER_NAME)
+        setup_container(
+            master_conn,
+            image_uri,
+            MASTER_CONTAINER_NAME,
+            entrypoint=container_entrypoint,
+            cmd=container_cmd,
+        )
+        setup_container(
+            worker_conn,
+            image_uri,
+            WORKER_CONTAINER_NAME,
+            entrypoint=container_entrypoint,
+            cmd=container_cmd,
+        )
 
         setup_master_ssh(master_conn)
         worker_private_ip = get_private_ip(aws_session, worker_id)
 
@@ -105,6 +105,7 @@ jobs:
       build-change: ${{ steps.changes.outputs.build-change }}
       sanity-test-change: ${{ steps.changes.outputs.sanity-test-change }}
       telemetry-test-change: ${{ steps.changes.outputs.telemetry-test-change }}
+      efa-test-change: ${{ steps.changes.outputs.efa-test-change }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -135,6 +136,10 @@ jobs:
               - "test/sanity/**"
             telemetry-test-change:
               - "test/telemetry/**"
+            efa-test-change:
+              - "test/efa/**"
+              - ".github/scripts/efa/**"
+              - ".github/workflows/reusable-efa-tests.yml"
 
   # ============================================================
   # Build runtime image
@@ -355,12 +360,23 @@ jobs:
   # EFA integration test (2x p4d.24xlarge, NCCL over EFA)
   # ============================================================
   efa-test:
-    needs: [build-images, sanity-test, security-test, unit-test]
-    if: success()
+    needs: [check-changes, build-images, sanity-test, security-test, unit-test, load-config]
+    # Run when the docker context, EFA test code, or this workflow changed.
+    # Falls back to the prod image when build-images is skipped, mirroring
+    # sanity-test/single-gpu-test.
+    if: |
+      always() && !failure() && !cancelled() &&
+      (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.efa-test-change == 'true') &&
+      (needs.build-images.result == 'success' || needs.build-images.result == 'skipped')
+    # Per-caller queue: new pushes to this PR cancel its own pending efa-test;
+    # different PRs / workflows don't displace each other on the global lock.
+    concurrency:
+      group: ${{ github.workflow }}-efa-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
     uses: ./.github/workflows/reusable-efa-tests.yml
     with:
-      image-uri: ${{ needs.build-images.outputs.runtime-image-uri }}
-      aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }}
+      image-uri: ${{ needs.build-images.result == 'success' && needs.build-images.outputs.runtime-image-uri || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, needs.load-config.outputs.prod-image) }}
+      aws-account-id: ${{ needs.build-images.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }}
       aws-region: ${{ vars.AWS_REGION }}
 
   # ============================================================
 
@@ -8,11 +8,13 @@ on:
     paths:
       - ".github/config/image/vllm-ec2.yml"
       - ".github/workflows/pr-vllm-ec2.yml"
+      - ".github/workflows/reusable-efa-tests.yml"
       - ".github/workflows/reusable-vllm-upstream-tests.yml"
       - "docker/vllm/Dockerfile"
       - "scripts/common/**"
       - "scripts/telemetry/**"
       - "scripts/vllm/**"
+      - "test/efa/**"
       - "test/sanity/**"
       - "test/telemetry/**"
       - "test/vllm/**"
@@ -102,6 +104,7 @@ jobs:
       build-change: ${{ steps.changes.outputs.build-change }}
       sanity-test-change: ${{ steps.changes.outputs.sanity-test-change }}
       telemetry-test-change: ${{ steps.changes.outputs.telemetry-test-change }}
+      efa-test-change: ${{ steps.changes.outputs.efa-test-change }}
     steps:
       - name: Checkout DLC source
         uses: actions/checkout@v5
@@ -133,6 +136,9 @@ jobs:
               - "test/sanity/**"
             telemetry-test-change:
               - "test/telemetry/**"
+            efa-test-change:
+              - ".github/workflows/reusable-efa-tests.yml"
+              - "test/efa/**"
 
   build-image:
     needs: [check-changes, load-config]
@@ -241,3 +247,33 @@ jobs:
       setup-script: test/vllm/scripts/vllm_test_setup.sh
       example-test-script: test/vllm/scripts/vllm_ec2_examples_test.sh
     secrets: inherit
+
+  # ============================================================
+  # EFA integration test (2x p4d.24xlarge, NCCL over EFA).
+  # Reuses the shared EFA test (test/efa/test_efa.py); the test's
+  # setup_nccl_tests.sh builds nccl-tests in-container against the
+  # nvidia-nccl-cu12 wheel that ships in vllm/vllm-openai.
+  # Runs when the image was rebuilt OR when EFA test files changed
+  # (in which case it tests against the prod image).
+  # ============================================================
+  efa-test:
+    needs: [check-changes, build-image, sanity-test, security-test, load-config]
+    if: |
+      always() && !failure() && !cancelled() &&
+      (needs.build-image.result == 'success' || needs.check-changes.outputs.efa-test-change == 'true')
+    # Per-caller queue: new pushes to this PR cancel its own pending efa-test;
+    # different PRs / workflows don't displace each other on the global lock.
+    concurrency:
+      group: ${{ github.workflow }}-efa-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    uses: ./.github/workflows/reusable-efa-tests.yml
+    with:
+      image-uri: ${{ needs.build-image.result == 'success' && needs.build-image.outputs.ci-image || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, needs.load-config.outputs.prod-image) }}
+      aws-account-id: ${{ needs.build-image.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }}
+      aws-region: ${{ vars.AWS_REGION }}
+      run-nixl-tests: true
+      # vLLM's dockerd_entrypoint.sh execs `vllm serve "$@"` and would parse
+      # any CMD as a model tag — override entrypoint to bash and keep the
+      # container alive for docker exec.
+      container-entrypoint: /bin/bash
+      container-cmd: -c 'sleep infinity'
@@ -23,6 +23,29 @@ on:
         required: false
         type: string
         default: 'p4d.24xlarge'
+      run-nixl-tests:
+        description: 'Run NIXL libfabric smoke + disaggregated PD tests (vLLM only)'
+        required: false
+        type: boolean
+        default: false
+      container-entrypoint:
+        description: |
+          Override the test container's entrypoint (passed to `docker run --entrypoint`).
+          Leave blank to use the image's baked-in entrypoint (PyTorch default).
+          Set to `/bin/bash` for images whose entrypoint can't accept a generic
+          shell arg (e.g., vLLM's `vllm serve "$@"`).
+        required: false
+        type: string
+        default: ''
+      container-cmd:
+        description: |
+          CMD passed to the test container. Defaults to `bash` (works with the
+          image's entrypoint + `-id` stdin). When `container-entrypoint` is set
+          to `/bin/bash`, this becomes argv to bash and must be an explicit
+          payload such as `-c 'sleep infinity'`.
+        required: false
+        type: string
+        default: 'bash'
 
 # Serialize EFA tests across all PRs to avoid p4d capacity contention.
 # Only one EFA test runs at a time globally; subsequent runs queue (no cancel).
@@ -48,9 +71,16 @@ jobs:
           uv pip install -r test/requirements.txt
 
       - name: Run EFA integration test
+        env:
+          TEST_IMAGE_URI: ${{ inputs.image-uri }}
+          EFA_INSTANCE_TYPE: ${{ inputs.efa-instance-type }}
+          RUN_NIXL_TESTS: ${{ inputs.run-nixl-tests && '1' || '0' }}
+          # Quote-safe via env: container-cmd may contain spaces and quotes
+          # (e.g., `-c 'sleep infinity'`), which would tokenize incorrectly
+          # if assigned inline on the shell command line.
+          EFA_CONTAINER_ENTRYPOINT: ${{ inputs.container-entrypoint }}
+          EFA_CONTAINER_CMD: ${{ inputs.container-cmd }}
         run: |
           source .venv/bin/activate
           PYTHONPATH=$(pwd)/test:$(pwd)/.github/scripts:$PYTHONPATH \
-          TEST_IMAGE_URI=${{ inputs.image-uri }} \
-          EFA_INSTANCE_TYPE=${{ inputs.efa-instance-type }} \
           python -m pytest test/efa/test_efa.py -vs -rA --tb=long
@@ -6,10 +6,10 @@ set -ex
 NUM_HOSTS_FILE=$1
 NUM_HOSTS=$2
 
-if [[ -z "${CUDA_HOME}" ]]; then
-    echo "CUDA_HOME variable is empty, please define it in dockerfile"
-    exit 1
-fi
+# Default CUDA_HOME for images that don't export it (vLLM Ubuntu).
+# PyTorch DLCs already set this in the Dockerfile so this is a no-op there.
+: "${CUDA_HOME:=/usr/local/cuda}"
+export CUDA_HOME
 
 TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
 INSTANCE_TYPE=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/instance-type)
@@ -52,6 +52,28 @@ check_efa_nccl_all_reduce_performance(){
     fi
 }
 
+# Capture diagnostics to a file we cat at the very end. invoke/Fabric truncate
+# the .stdout of a failing remote command to the last few KB, so anything
+# printed before mpirun gets dropped. Stage it through a file and dump after
+# the validators run.
+DIAG_LOG="/test/efa/logs/diagnostics.log"
+{
+    echo "==================== EFA / NCCL diagnostics ===================="
+    echo "--- nvidia-smi ---"
+    nvidia-smi -L || true
+    echo "--- libnccl resolution ---"
+    ldconfig -p | grep libnccl || echo "(no libnccl in ldconfig)"
+    echo "--- ldd all_reduce_perf ---"
+    ldd /usr/local/bin/all_reduce_perf 2>&1 | grep -E "nccl|cuda|fabric|not found" || true
+    echo "--- libfabric provider list ---"
+    fi_info -p efa 2>&1 | head -20 || true
+    echo "--- aws-ofi-nccl plugin ---"
+    ls -la /opt/amazon/ofi-nccl/lib*/libnccl-net*.so 2>&1 | head -5 || true
+    echo "--- /etc/ld.so.conf.d ---"
+    ls /etc/ld.so.conf.d/ 2>&1
+    echo "==================== end diagnostics ===================="
+} > "${DIAG_LOG}" 2>&1
+
 echo "Running all_reduce_perf test"
 mpirun -x FI_PROVIDER="efa" -x FI_EFA_FORK_SAFE=1 -n $NODES -N $GPU_COUNT --hostfile $NUM_HOSTS_FILE \
     -x NCCL_DEBUG=INFO ${USE_DEVICE_RDMA_ARG} -x NCCL_PROTO=simple -x NCCL_ALGO=ring -x RDMAV_FORK_SAFE=1 \
@@ -66,5 +88,27 @@ else
     echo "check_efa_nccl_all_reduce failed"
 fi
 
+# Dump training log first (it can be huge on success and we don't need it on
+# failure — mpirun's stdout was already captured), then the most actionable
+# diagnostics LAST so they survive Fabric's stdout truncation.
+echo "==================== BEGIN ${TRAINING_LOG} ===================="
+cat "${TRAINING_LOG}" 2>/dev/null || echo "(log file missing)"
+echo "==================== END ${TRAINING_LOG} ===================="
+
+echo "==================== BEGIN ${DIAG_LOG} ===================="
+cat "${DIAG_LOG}" 2>/dev/null || echo "(diagnostics file missing)"
+echo "==================== END ${DIAG_LOG} ===================="
+
+# These are the smallest, highest-signal probes — placed last so the
+# truncated tail Fabric retains will always show them.
+echo "==================== final probes ===================="
+echo "--- ldd all_reduce_perf (libnccl resolution) ---"
+ldd /usr/local/bin/all_reduce_perf 2>&1 | grep -E "nccl|cuda|fabric|not found" || true
+echo "--- libnccl SONAMES on system ---"
+find / -name 'libnccl.so*' 2>/dev/null | head -10
+echo "--- aws-ofi-nccl plugin paths ---"
+ls -la /opt/amazon/ofi-nccl/lib*/libnccl-net*.so 2>&1
+echo "==================== end ===================="
+
 validate_all_reduce_performance_logs
 check_efa_nccl_all_reduce_performance