Merge remote-tracking branch 'origin/main' into sj/fine-grained-activation-offload

Author: seonjinn

.agents/skills

@@ -0,0 +1 @@
+../skills

.claude/settings.json

@@ -0,0 +1,14 @@
+{
+  "hooks": {
+    "UserPromptSubmit": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "printf '{\"hookSpecificOutput\":{\"hookEventName\":\"UserPromptSubmit\",\"additionalContext\":\"MANDATORY WORKFLOW — never skip or reorder: (1) Read the artifact first (commit, file, error, PR). (2) Identify and invoke the relevant skill via the Skill tool BEFORE forming any answer or plan — even when the answer seems obvious. (3) Only then answer using the skill context. Skipping step 2 is not allowed.\"}}'"
+          }
+        ]
+      }
+    ]
+  }
+}

.claude/skills

@@ -0,0 +1 @@
+../skills

.github/actions/test-template/action.yml

@@ -81,14 +81,21 @@ runs:
       if: ${{ inputs.has-azure-credentials == 'true' }}
       shell: bash
       run: |
-        curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+        for i in 1 2 3; do
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash && break
+          echo "Attempt $i failed, retrying in 10s..."
+          sleep 10
+        done
 
     - name: Install uuidgen
       shell: bash -x -e -u -o pipefail {0}
       if: ${{ contains(inputs.runner, 'gcp') }}
       run: |
-        apt-get update
-        apt-get install -y uuid-runtime
+        for i in 1 2 3; do
+          apt-get update && apt-get install -y uuid-runtime && break
+          echo "Attempt $i failed, retrying in 10s..."
+          sleep 10
+        done
 
     - name: Docker system cleanup
       shell: bash
@@ -122,6 +129,12 @@ runs:
         COVERAGE_PREFIX=$([[ "${{ inputs.is_doc_test }}" == "true" ]] && echo "doc-test" || ([[ "${{ inputs.is_unit_test }}" == "true" ]] && echo "unit-test" || echo "e2e"))
         echo "coverage-prefix=$COVERAGE_PREFIX" | tee -a "$GITHUB_OUTPUT"
 
+        echo -e "\033[1;34m┌─ launching test ─────────────────────────────────────────────────────────┐\033[0m"
+        echo -e "\033[1;34m│  script    : ${{ inputs.script }}\033[0m"
+        echo -e "\033[1;34m│  runner    : ${{ inputs.runner }}\033[0m"
+        echo -e "\033[1;34m│  container : ${{ inputs.registry }}/${{ inputs.image }}:${{ inputs.image-tag || github.run_id }}\033[0m"
+        echo -e "\033[1;34m└──────────────────────────────────────────────────────────────────────────┘\033[0m"
+        echo "::group::Logs"
         docker run --rm -u root --runtime=nvidia --gpus all \
           --shm-size=64g \
           --env TRANSFORMERS_OFFLINE=0 \
@@ -152,28 +165,43 @@ runs:
             fi
             bash tests/${{ inputs.is_doc_test == 'true' && 'docs' || (inputs.is_unit_test == 'true' && 'unit' || 'functional') }}/${{ inputs.script }}.sh && \
             echo "Finished successfully." || echo "Did not finish."' 2>&1 | tee err.log
+        echo "::endgroup::"
 
     - name: Check result
       id: check
-      shell: bash
+      shell: bash -e -u -o pipefail {0}
+      if: always()
       run: |
         coverage_report=coverage-${{ steps.test.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
-        echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
+        echo "coverage_report=$coverage_report" | tee -a "$GITHUB_OUTPUT"
 
         IS_SUCCESS=$(tail -n 1 err.log | grep -q "Finished successfully." && echo "true" || echo "false")
 
-        if [[ "$IS_SUCCESS" == "false" && "{% raw %}${{ inputs.is_optional }}" == "true" ]]; then
-          echo "::warning:: Test failed, but displayed as successful because it is marked as optional."
+        if [[ "$IS_SUCCESS" == "false" && "${{ inputs.is_optional }}" == "true" ]]; then
+          echo "::warning::Test failed but is marked optional — treating as success."
           IS_SUCCESS=true
         fi
 
-        if [[ "$IS_SUCCESS" == "false" ]]; then
-          echo Test did not finish successfully.
+        if [[ "$IS_SUCCESS" == "true" ]]; then
+          echo -e "\033[1;32m╔══════════════════════════════════════════════════════════════════════════╗\033[0m"
+          echo -e "\033[1;32m║                                                                          ║\033[0m"
+          echo -e "\033[1;32m║   ✅  PASSED                                                             ║\033[0m"
+          echo -e "\033[1;32m║   ${{ inputs.script }}\033[0m"
+          echo -e "\033[1;32m║                                                                          ║\033[0m"
+          echo -e "\033[1;32m╚══════════════════════════════════════════════════════════════════════════╝\033[0m"
+          echo "::notice title=Result::✅ ${{ inputs.script }} — PASSED"
+          exit 0
+        else
+          echo -e "\033[1;31m╔══════════════════════════════════════════════════════════════════════════╗\033[0m"
+          echo -e "\033[1;31m║                                                                          ║\033[0m"
+          echo -e "\033[1;31m║   ❌  FAILED                                                             ║\033[0m"
+          echo -e "\033[1;31m║   ${{ inputs.script }}\033[0m"
+          echo -e "\033[1;31m║                                                                          ║\033[0m"
+          echo -e "\033[1;31m╚══════════════════════════════════════════════════════════════════════════╝\033[0m"
+          echo "::error title=Result::❌ ${{ inputs.script }} — FAILED"
           exit 1
         fi
 
-        exit $EXIT_CODE
-
     - name: Upload artifacts
       uses: actions/upload-artifact@v6
       if: ${{ steps.check.outputs.coverage_report != 'none' }}
@@ -183,6 +211,15 @@ runs:
           ${{ github.workspace }}/${{ github.run_id }}/${{steps.uuid.outputs.id }}/nemo-rl/tests/.coverage
         include-hidden-files: true
 
+    - name: Upload nemo_gym actual test data
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: actual_test_nemo_gym_sanity-${{ github.run_id }}
+        path: |
+          ${{ github.workspace }}/${{ github.run_id }}/${{steps.uuid.outputs.id }}/nemo-rl/tests/unit/environments/nemo_gym_test_data/actual_test_nemo_gym_sanity.json
+        if-no-files-found: ignore
+
     - name: Container shutdown
       if: always()
       shell: bash

.github/workflows/build-test-publish-wheel.yml

@@ -39,7 +39,7 @@ on:
         default: ""
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }}
   cancel-in-progress: true
 
 jobs:
@@ -291,16 +291,50 @@ jobs:
       build-ref: ${{ needs.pre-flight.outputs.test_sha }}
       image-name: ${{ vars.CI_CONTAINER_NAME }}
       dockerfile: docker/Dockerfile
-      runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}-gpu-x2
+      runner: ${{ contains(needs.org-member-pre-flight.outputs.runner_prefix, 'azure') && format('{0}-gpu-x2', needs.org-member-pre-flight.outputs.runner_prefix) || contains(needs.org-member-pre-flight.outputs.runner_prefix, 'gcp') && format('{0}-gpu-x4', needs.org-member-pre-flight.outputs.runner_prefix) }}
       image-label: ${{ vars.CI_CONTAINER_NAME }}
       target: release
       registry: ${{ needs.org-member-pre-flight.outputs.registry }}
       build-contexts: |
         nemo-rl=${{ github.run_id }}/
+        ${{ vars.UV_BUILD_CACHE == 'enabled' && format('uv-cache-seed=docker-image://{0}/{1}-uv-cache:latest', needs.org-member-pre-flight.outputs.registry, vars.CI_CONTAINER_NAME) || '' }}
       build-args: |
         MAX_JOBS=4
         NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }}
 
+  update-uv-cache:
+    name: Update uv build cache
+    needs: [build-container, org-member-pre-flight]
+    if: >-
+      ${{
+        github.ref == 'refs/heads/main' &&
+        needs.build-container.result == 'success'
+      }}
+    runs-on: ${{ format('{0}-gpu-x2', needs.org-member-pre-flight.outputs.runner_prefix) }}
+    environment: nemo-ci
+    env:
+      REGISTRY: ${{ needs.org-member-pre-flight.outputs.registry }}
+      IMAGE_NAME: ${{ vars.CI_CONTAINER_NAME }}
+    steps:
+      - name: Extract and push uv cache image
+        run: |
+          set -euo pipefail
+          SRC="${REGISTRY}/${IMAGE_NAME}:${{ github.run_id }}"
+          DST="${REGISTRY}/${IMAGE_NAME}-uv-cache:latest"
+
+          docker pull "${SRC}"
+          CID=$(docker create "${SRC}" true)
+          mkdir -p /tmp/uv-cache
+          docker cp "${CID}:/root/.cache/uv/." /tmp/uv-cache/
+          docker rm "${CID}"
+
+          printf 'FROM scratch\nCOPY uv-cache/ /\n' > /tmp/Dockerfile.uv-cache
+          docker build -t "${DST}" -f /tmp/Dockerfile.uv-cache /tmp
+          docker push "${DST}"
+
+          docker rmi "${SRC}" "${DST}" 2>/dev/null || true
+          rm -rf /tmp/uv-cache /tmp/Dockerfile.uv-cache
+
   cicd-doc-tests:
     strategy:
       fail-fast: false
@@ -480,6 +514,24 @@ jobs:
           SUMMARY=$(echo $JOB_RESULTS | jq 'to_entries[] | .key + ": " + .value.result' | tr -d '"')
           echo '🤖: CICD Result for test level: ${{ needs.pre-flight.outputs.test_level }}' >> $GITHUB_STEP_SUMMARY
           echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY
+
+          if [[ "$TEST_LEVEL" == "none" ]]; then
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "---" >> $GITHUB_STEP_SUMMARY
+            echo "⚠️ **No tests were run.** This PR does not have a CI label." >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "To trigger tests, add one of the following labels to your PR:" >> $GITHUB_STEP_SUMMARY
+            echo "| Label | What it runs |" >> $GITHUB_STEP_SUMMARY
+            echo "|-------|-------------|" >> $GITHUB_STEP_SUMMARY
+            echo "| \`CI:docs\` | Doc tests only |" >> $GITHUB_STEP_SUMMARY
+            echo "| \`CI:Lfast\` | Fast subset (reuses main container) |" >> $GITHUB_STEP_SUMMARY
+            echo "| \`CI:L0\` | Unit tests + docs + lint |" >> $GITHUB_STEP_SUMMARY
+            echo "| \`CI:L1\` | L0 + functional tests |" >> $GITHUB_STEP_SUMMARY
+            echo "| \`CI:L2\` | L1 + convergence tests |" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "This check will remain failed until a CI label is added." >> $GITHUB_STEP_SUMMARY
+          fi
+
           test "$ALL_SUCCESS" = "true" || test "$CI_SKIP" = "true"
 
   notify-nightly-failure:
@@ -519,19 +571,25 @@ jobs:
       matrix:
         flag: [doc-test, unit-test, e2e]
     steps:
+      - name: Get PR info
+        id: get-pr-info
+        if: startsWith(github.ref, 'refs/heads/pull-request/')
+        uses: nv-gha-runners/get-pr-info@main
+
       - name: Checkout
         uses: actions/checkout@v6
 
       - name: Download coverage reports of current branch
         uses: actions/download-artifact@v7
         with:
           pattern: coverage-${{ matrix.flag }}-*
+          path: coverage-downloads
 
       - name: Check if artifacts were downloaded
         id: check-artifacts
         run: |
-          # Check if any coverage directories were downloaded
-          if ls coverage-* 1> /dev/null 2>&1; then
+          # Check if any .coverage files were downloaded
+          if find coverage-downloads -name ".coverage" -type f 2>/dev/null | grep -q .; then
             echo "artifacts-found=true" >> $GITHUB_OUTPUT
             echo "Found coverage artifacts for ${{ matrix.flag }}"
           else
@@ -545,12 +603,10 @@ jobs:
         run: |
           pip install coverage
 
-          ls -al .
-          ls -al coverage-*/
-          coverage combine --keep $(ls coverage-*/.coverage)
+          find coverage-downloads -name ".coverage" -type f
+          coverage combine --keep $(find coverage-downloads -name ".coverage" -type f)
           coverage report -i --show-missing
-          rm -rf coverage-*
-          ls -al
+          rm -rf coverage-downloads
 
       - name: Skip coverage processing
         if: ${{ steps.check-artifacts.outputs.artifacts-found == 'false' }}
@@ -564,6 +620,7 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
           flags: ${{ matrix.flag }}
+          base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').base.sha }}
 
       - name: Upload artifacts
         if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }}

.github/workflows/cicd-main.yml

@@ -16,9 +16,15 @@ jobs:
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_claude_review.yml@v0.79.0
     with:
       prompt: |
-        You are doing a light code review. Keep it concise and actionable.
+        Mandatory workflow — never skip or reorder:
+        1. Read the PR diff first.
+        2. Based on the changed files and areas, identify relevant skills from skills/<name>/SKILL.md.
+           Common skill names: linting-and-formatting, testing, cicd, build-and-dependency,
+           contributing, copyright, docs, error-handling, config-conventions, launch-nemo-rl.
+        3. Read the SKILL.md files for all relevant areas using the Read tool.
+        4. Only then perform the review using the skill context.
 
-        Read and follow the review guidelines in CLAUDE.md and the coding guideline skills in .claude/skills/ at the repository root.
+        You are doing a light code review. Keep it concise and actionable.
 
         Focus ONLY on:
         - Critical bugs or logic errors