Merge branch 'develop' into antoiner/feat/ovphysx_rigidobject

Author: AntoineRichard

.github/actions/run-package-tests/action.yml

@@ -55,6 +55,10 @@ inputs:
     description: 'Additional pytest options'
     default: ''
     required: false
+  extra-pip-packages:
+    description: 'Space-separated pip packages to install inside the Docker container before pytest starts'
+    default: ''
+    required: false
   container-name:
     description: 'Docker container name prefix (run-id is appended automatically)'
     required: true
@@ -83,6 +87,15 @@ runs:
           echo "🟠 Could not obtain IMDS token, probably not running on EC2"
         fi
 
+    - name: Prepare Docker disk space
+      shell: bash
+      run: |
+        bash .github/actions/run-package-tests/cleanup_docker_storage.sh \
+          "${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \
+          "isaacsim-pin-${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}-pre" \
+          "pre-test-cleanup" \
+          "true"
+
     - name: Record pull start time
       id: pull-start
       shell: bash
@@ -125,6 +138,7 @@ runs:
         quarantined-only: ${{ inputs.quarantined-only }}
         include-files: ${{ inputs.include-files }}
         volume-mount-source: ${{ github.workspace }}
+        extra-pip-packages: ${{ inputs.extra-pip-packages }}
 
     - name: Check Test Results
       if: always()
@@ -153,28 +167,8 @@ runs:
       if: always()
       shell: bash
       run: |
-        # Don't let cleanup errors fail a passing test job
-        set +e
-
-        echo "🔵 Cleaning up old Docker images..."
-
-        echo "⚪ Disk usage before cleanup:"
-        df -h /var/lib/docker
-
-        ISAACSIM_IMAGE="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}"
-
-        # Pin the base IsaacSim image so prune -a doesn't remove it
-        ISAACSIM_IMAGE_PIN_CONTAINER="isaacsim-pin-${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}"
-        echo "🔵 Pinning IsaacSim base image: ${ISAACSIM_IMAGE} to temporary container ${ISAACSIM_IMAGE_PIN_CONTAINER}"
-        docker create --name "${ISAACSIM_IMAGE_PIN_CONTAINER}" "${ISAACSIM_IMAGE}" true 2>/dev/null || true
-
-        echo "🔵 Removing Docker images older than 3 days..."
-        docker image prune -a -f --filter "until=72h"
-
-        echo "🔵 Removing temporary container ${ISAACSIM_IMAGE_PIN_CONTAINER}..."
-        docker rm "${ISAACSIM_IMAGE_PIN_CONTAINER}" 2>/dev/null || true
-
-        echo "🟢 Docker image cleanup complete."
-
-        echo "⚪ Disk usage after cleanup:"
-        df -h /var/lib/docker
+        bash .github/actions/run-package-tests/cleanup_docker_storage.sh \
+          "${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \
+          "isaacsim-pin-${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}-post" \
+          "post-job-cleanup" \
+          "false"

.github/actions/run-package-tests/cleanup_docker_storage.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -u
+
+ISAACSIM_IMAGE="${1:-}"
+ISAACSIM_IMAGE_PIN_CONTAINER="${2:-isaacsim-pin-run-package-tests}"
+CLEANUP_CONTEXT="${3:-docker-cleanup}"
+FAIL_IF_STILL_HIGH="${4:-false}"
+
+DOCKER_STORAGE_PATH="${DOCKER_STORAGE_PATH:-/var/lib/docker}"
+DOCKER_CLEANUP_THRESHOLD_PERCENT="${DOCKER_CLEANUP_THRESHOLD_PERCENT:-85}"
+DOCKER_AGGRESSIVE_PRUNE_UNTIL="${DOCKER_AGGRESSIVE_PRUNE_UNTIL:-24h}"
+DOCKER_CONSERVATIVE_PRUNE_UNTIL="${DOCKER_CONSERVATIVE_PRUNE_UNTIL:-72h}"
+
+get_docker_storage_usage_percent() {
+    df -P "${DOCKER_STORAGE_PATH}" 2>/dev/null | awk 'NR == 2 { gsub("%", "", $5); print $5 }'
+}
+
+resolve_docker_storage_path() {
+    local docker_root_dir
+    docker_root_dir="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
+    if [ -n "${docker_root_dir}" ]; then
+        DOCKER_STORAGE_PATH="${docker_root_dir}"
+        echo "[${CLEANUP_CONTEXT}] Using Docker root from docker info: ${DOCKER_STORAGE_PATH}"
+        return 0
+    fi
+
+    echo "[${CLEANUP_CONTEXT}] Could not determine Docker storage path from ${DOCKER_STORAGE_PATH} or docker info." >&2
+    return 1
+}
+
+print_docker_storage_usage() {
+    echo "[${CLEANUP_CONTEXT}] Docker storage usage:"
+    df -h "${DOCKER_STORAGE_PATH}" || true
+    docker system df || true
+}
+
+pin_isaacsim_image_if_present() {
+    if [ -z "${ISAACSIM_IMAGE}" ]; then
+        echo "[${CLEANUP_CONTEXT}] No IsaacSim base image provided; skipping image pin."
+        return
+    fi
+
+    if docker image inspect "${ISAACSIM_IMAGE}" >/dev/null 2>&1; then
+        echo "[${CLEANUP_CONTEXT}] Pinning IsaacSim base image: ${ISAACSIM_IMAGE}"
+        docker create --name "${ISAACSIM_IMAGE_PIN_CONTAINER}" "${ISAACSIM_IMAGE}" true >/dev/null 2>&1 || true
+    else
+        echo "[${CLEANUP_CONTEXT}] IsaacSim base image is not present locally; skipping image pin."
+    fi
+}
+
+remove_isaacsim_image_pin() {
+    docker rm "${ISAACSIM_IMAGE_PIN_CONTAINER}" >/dev/null 2>&1 || true
+}
+
+run_conservative_cleanup() {
+    echo "[${CLEANUP_CONTEXT}] Running conservative Docker cleanup."
+    echo "[${CLEANUP_CONTEXT}] Removing Docker images older than ${DOCKER_CONSERVATIVE_PRUNE_UNTIL}."
+    docker image prune -a -f --filter "until=${DOCKER_CONSERVATIVE_PRUNE_UNTIL}" || true
+}
+
+run_aggressive_cleanup() {
+    echo "[${CLEANUP_CONTEXT}] Running aggressive Docker cleanup."
+    echo "[${CLEANUP_CONTEXT}] Removing stopped Docker containers."
+    docker container prune -f --filter "until=${DOCKER_AGGRESSIVE_PRUNE_UNTIL}" || true
+
+    echo "[${CLEANUP_CONTEXT}] Removing unused Docker builder cache older than ${DOCKER_AGGRESSIVE_PRUNE_UNTIL}."
+    docker builder prune -a -f --filter "until=${DOCKER_AGGRESSIVE_PRUNE_UNTIL}" || true
+
+    echo "[${CLEANUP_CONTEXT}] Removing Docker images older than ${DOCKER_AGGRESSIVE_PRUNE_UNTIL}."
+    docker image prune -a -f --filter "until=${DOCKER_AGGRESSIVE_PRUNE_UNTIL}" || true
+}
+
+main() {
+    echo "[${CLEANUP_CONTEXT}] Checking Docker storage before cleanup."
+
+    local usage_percent
+    usage_percent="$(get_docker_storage_usage_percent)"
+    if [ -z "${usage_percent}" ]; then
+        resolve_docker_storage_path || true
+        usage_percent="$(get_docker_storage_usage_percent)"
+    fi
+
+    print_docker_storage_usage
+
+    if [ -z "${usage_percent}" ]; then
+        echo "[${CLEANUP_CONTEXT}] Could not determine Docker storage usage; running conservative cleanup."
+        pin_isaacsim_image_if_present
+        trap remove_isaacsim_image_pin EXIT
+        run_conservative_cleanup
+        print_docker_storage_usage
+        if [ "${FAIL_IF_STILL_HIGH}" = "true" ]; then
+            echo "::error::Could not determine Docker storage usage after cleanup. The self-hosted runner needs a measurable Docker storage path before tests can run." >&2
+            return 1
+        fi
+        return 0
+    fi
+
+    pin_isaacsim_image_if_present
+    trap remove_isaacsim_image_pin EXIT
+
+    if [ "${usage_percent}" -ge "${DOCKER_CLEANUP_THRESHOLD_PERCENT}" ]; then
+        echo "[${CLEANUP_CONTEXT}] Docker storage is at ${usage_percent}%, meeting the ${DOCKER_CLEANUP_THRESHOLD_PERCENT}% threshold."
+        run_aggressive_cleanup
+    else
+        echo "[${CLEANUP_CONTEXT}] Docker storage is at ${usage_percent}%, below the ${DOCKER_CLEANUP_THRESHOLD_PERCENT}% threshold."
+        run_conservative_cleanup
+    fi
+
+    echo "[${CLEANUP_CONTEXT}] Docker storage after cleanup."
+    print_docker_storage_usage
+
+    local final_usage_percent
+    final_usage_percent="$(get_docker_storage_usage_percent)"
+    if [ "${FAIL_IF_STILL_HIGH}" = "true" ] \
+        && [ -n "${final_usage_percent}" ] \
+        && [ "${final_usage_percent}" -ge "${DOCKER_CLEANUP_THRESHOLD_PERCENT}" ]; then
+        echo "::error::Docker storage remains at ${final_usage_percent}% after cleanup. The self-hosted runner needs more free Docker storage before tests can run." >&2
+        return 1
+    fi
+}
+
+main "$@"

.github/actions/run-package-tests/test_cleanup_docker_storage.py

@@ -0,0 +1,181 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Tests for CI Docker storage cleanup helpers."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+_ACTION_DIR = Path(__file__).resolve().parent
+_REPO_ROOT = _ACTION_DIR.parents[2]
+_ACTION_PATH = _ACTION_DIR / "action.yml"
+_SCRIPT_PATH = _ACTION_DIR / "cleanup_docker_storage.sh"
+
+
+def _write_fake_df(
+    bin_dir: Path, first_usage: int, final_usage: int, failed_path: str | None = None, fallback_path: str | None = None
+) -> None:
+    count_file = bin_dir / "df-count"
+    count_file.write_text("0", encoding="utf-8")
+    script = bin_dir / "df"
+    script.write_text(
+        f"""#!/usr/bin/env bash
+set -euo pipefail
+
+path="${{2:-}}"
+if [ -n "{failed_path or ""}" ] && [ "$path" = "{failed_path or ""}" ]; then
+    exit 1
+fi
+if [ -n "{fallback_path or ""}" ] && [ "$path" = "{fallback_path or ""}" ]; then
+    :
+elif [ -n "{fallback_path or ""}" ]; then
+    exit 1
+fi
+
+if [ "${{1:-}}" = "-P" ]; then
+    count=$(cat "{count_file}")
+    if [ "$count" -eq 0 ]; then
+        usage="{first_usage}"
+    else
+        usage="{final_usage}"
+    fi
+    echo "$((count + 1))" > "{count_file}"
+    echo "Filesystem 1024-blocks Used Available Capacity Mounted on"
+    echo "/dev/test 100 90 10 ${{usage}}% /var/lib/docker"
+else
+    echo "Filesystem Size Used Avail Use% Mounted on"
+    echo "/dev/test 100G 90G 10G {first_usage}% /var/lib/docker"
+fi
+""",
+        encoding="utf-8",
+    )
+    script.chmod(0o755)
+
+
+def _write_fake_docker(bin_dir: Path, docker_root_dir: str = "/var/lib/docker") -> Path:
+    log_path = bin_dir / "docker.log"
+    script = bin_dir / "docker"
+    script.write_text(
+        f"""#!/usr/bin/env bash
+set -euo pipefail
+
+printf '%s\\n' "$*" >> "{log_path}"
+if [ "${{1:-}}" = "info" ] && [ "${{2:-}}" = "--format" ]; then
+    if [ -z "{docker_root_dir}" ]; then
+        exit 1
+    fi
+    echo "{docker_root_dir}"
+    exit 0
+fi
+if [ "${{1:-}}" = "image" ] && [ "${{2:-}}" = "inspect" ]; then
+    exit 0
+fi
+exit 0
+""",
+        encoding="utf-8",
+    )
+    script.chmod(0o755)
+    return log_path
+
+
+def _run_cleanup(
+    tmp_path: Path,
+    first_usage: int,
+    final_usage: int,
+    failed_path: str | None = None,
+    docker_root_dir: str = "/var/lib/docker",
+) -> subprocess.CompletedProcess[str]:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    _write_fake_df(bin_dir, first_usage, final_usage, failed_path=failed_path, fallback_path=docker_root_dir)
+    _write_fake_docker(bin_dir, docker_root_dir=docker_root_dir)
+
+    env = os.environ.copy()
+    env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+    env["DOCKER_STORAGE_PATH"] = "/var/lib/docker"
+    env["DOCKER_CLEANUP_THRESHOLD_PERCENT"] = "85"
+
+    return subprocess.run(
+        [
+            "bash",
+            str(_SCRIPT_PATH),
+            "nvcr.io/nvidian/isaac-sim:latest-develop",
+            "isaacsim-pin-test",
+            "test-cleanup",
+            "true",
+        ],
+        cwd=_REPO_ROOT,
+        env=env,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+
+def test_cleanup_uses_aggressive_pruning_when_docker_storage_exceeds_threshold(tmp_path: Path):
+    result = _run_cleanup(tmp_path, first_usage=91, final_usage=70)
+
+    assert result.returncode == 0, result.stderr
+    docker_log = (tmp_path / "bin" / "docker.log").read_text(encoding="utf-8")
+    assert "container prune -f --filter until=24h" in docker_log
+    assert "builder prune -a -f --filter until=24h" in docker_log
+    assert "image prune -a -f --filter until=24h" in docker_log
+
+
+def test_cleanup_keeps_conservative_image_pruning_below_threshold(tmp_path: Path):
+    result = _run_cleanup(tmp_path, first_usage=70, final_usage=70)
+
+    assert result.returncode == 0, result.stderr
+    docker_log = (tmp_path / "bin" / "docker.log").read_text(encoding="utf-8")
+    assert "image prune -a -f --filter until=72h" in docker_log
+    assert "builder prune" not in docker_log
+
+
+def test_cleanup_fails_early_when_aggressive_pruning_cannot_recover_space(tmp_path: Path):
+    result = _run_cleanup(tmp_path, first_usage=91, final_usage=89)
+
+    assert result.returncode == 1
+    assert "Docker storage remains at 89%" in result.stderr
+
+
+def test_cleanup_uses_docker_root_dir_when_default_storage_path_is_unavailable(tmp_path: Path):
+    result = _run_cleanup(
+        tmp_path,
+        first_usage=91,
+        final_usage=89,
+        failed_path="/var/lib/docker",
+        docker_root_dir="/mnt/docker-root",
+    )
+
+    assert result.returncode == 1
+    assert "Using Docker root from docker info: /mnt/docker-root" in result.stdout
+    assert "Docker storage remains at 89%" in result.stderr
+
+
+def test_cleanup_fails_early_when_docker_storage_usage_cannot_be_measured(tmp_path: Path):
+    result = _run_cleanup(
+        tmp_path,
+        first_usage=91,
+        final_usage=89,
+        failed_path="/var/lib/docker",
+        docker_root_dir="",
+    )
+
+    assert result.returncode == 1
+    assert "Could not determine Docker storage usage after cleanup" in result.stderr
+
+
+def test_run_package_tests_prepares_docker_space_before_image_pull():
+    action_text = _ACTION_PATH.read_text(encoding="utf-8")
+
+    cleanup_index = action_text.index("name: Prepare Docker disk space")
+    pull_start_index = action_text.index("name: Record pull start time")
+    pull_index = action_text.index("name: Pull image from ECR")
+
+    assert cleanup_index < pull_start_index < pull_index
+    assert "cleanup_docker_storage.sh" in action_text