From d8749aad237e89cd8cee7ecc12a2e109c9555dde Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <13862180016@163.com> Date: Wed, 22 Jul 2026 06:01:21 +0800 Subject: [PATCH 1/2] ci: add SAI GPU validation --- .github/workflows/sai-bootstrap.yml | 104 ++ .github/workflows/sai-gpu-full.yml | 323 +++++ ci/sai/README.md | 227 +++ ci/sai/build_gpu.sbatch | 14 + ci/sai/build_gpu.sh | 205 +++ ci/sai/cleanup_sai_runs.sh | 146 ++ ci/sai/collect_remote_artifacts.sh | 45 + ci/sai/configure_ssh_client.sh | 55 + ci/sai/install_cleanup_cron.sh | 116 ++ ci/sai/known_hosts | 1 + ci/sai/mark_artifacts_uploaded.sh | 18 + ci/sai/mpirun_with_mapping.sh | 8 + ci/sai/prepare_cleanup_install.sh | 60 + ci/sai/prepare_cusolvermp_smoke.sh | 48 + ci/sai/prepare_gpu_case_matrix.sh | 91 ++ ci/sai/prepare_nvidia_mp.sh | 114 ++ ci/sai/prepare_remote_run.sh | 77 + ci/sai/probe_remote_sai.sh | 14 + ci/sai/run_gpu_case_attempts.sh | 99 ++ ci/sai/run_gpu_case_matrix.sh | 210 +++ ci/sai/run_gpu_validation.sh | 88 ++ ci/sai/run_remote_ci.sh | 111 ++ ci/sai/run_slurm_job.sh | 107 ++ ci/sai/source_transfer_cache.sh | 371 +++++ ci/sai/summarize_gpu_case_matrix.sh | 164 +++ ci/sai/test_gpu.sbatch | 105 ++ ci/sai/test_gpu_case.sh | 107 ++ ci/sai/tests/test_sai_ci.sh | 1262 +++++++++++++++++ .../archive-mp09-sai-nccl2293.env.example | 43 + .../19_NO_Si48_CUSOLVERMP_TDDFT_GPU/INPUT | 37 + .../19_NO_Si48_CUSOLVERMP_TDDFT_GPU/KPT | 4 + .../19_NO_Si48_CUSOLVERMP_TDDFT_GPU/README | 8 + .../19_NO_Si48_CUSOLVERMP_TDDFT_GPU/STRU | 68 + 33 files changed, 4450 insertions(+) create mode 100644 .github/workflows/sai-bootstrap.yml create mode 100644 .github/workflows/sai-gpu-full.yml create mode 100644 ci/sai/README.md create mode 100755 ci/sai/build_gpu.sbatch create mode 100755 ci/sai/build_gpu.sh create mode 100755 ci/sai/cleanup_sai_runs.sh create mode 100755 ci/sai/collect_remote_artifacts.sh create mode 100755 ci/sai/configure_ssh_client.sh create mode 100755 ci/sai/install_cleanup_cron.sh create mode 100644 ci/sai/known_hosts create mode 100755 ci/sai/mark_artifacts_uploaded.sh create mode 100755 ci/sai/mpirun_with_mapping.sh create mode 100755 ci/sai/prepare_cleanup_install.sh create mode 100644 ci/sai/prepare_cusolvermp_smoke.sh create mode 100755 ci/sai/prepare_gpu_case_matrix.sh create mode 100755 ci/sai/prepare_nvidia_mp.sh create mode 100755 ci/sai/prepare_remote_run.sh create mode 100755 ci/sai/probe_remote_sai.sh create mode 100755 ci/sai/run_gpu_case_attempts.sh create mode 100755 ci/sai/run_gpu_case_matrix.sh create mode 100755 ci/sai/run_gpu_validation.sh create mode 100755 ci/sai/run_remote_ci.sh create mode 100755 ci/sai/run_slurm_job.sh create mode 100755 ci/sai/source_transfer_cache.sh create mode 100755 ci/sai/summarize_gpu_case_matrix.sh create mode 100755 ci/sai/test_gpu.sbatch create mode 100755 ci/sai/test_gpu_case.sh create mode 100755 ci/sai/tests/test_sai_ci.sh create mode 100644 ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example create mode 100644 tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/INPUT create mode 100644 tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/KPT create mode 100644 tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/README create mode 100644 tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/STRU diff --git a/.github/workflows/sai-bootstrap.yml b/.github/workflows/sai-bootstrap.yml new file mode 100644 index 00000000000..dde41304fcf --- /dev/null +++ b/.github/workflows/sai-bootstrap.yml @@ -0,0 +1,104 @@ +name: SAI SSH Bootstrap + +on: + workflow_dispatch: + inputs: + mode: + description: "Read-only probe or install the user cron cleanup" + required: true + default: probe + type: choice + options: + - probe + - install-cleanup + project_root: + description: "Optional absolute project root below the SAI account HOME" + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + bootstrap: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: + name: sai-ssh-manual + env: + SAI_PROJECT_ROOT_DEFAULT: ${{ vars.SAI_PROJECT_ROOT }} + SAI_SSH_HOST: ${{ vars.SAI_SSH_HOST }} + SAI_SSH_PORT: ${{ vars.SAI_SSH_PORT }} + SAI_SSH_USER: ${{ vars.SAI_SSH_USER }} + steps: + - name: Checkout trusted control plane + uses: actions/checkout@v6 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Configure pinned SSH client + env: + SAI_SSH_PRIVATE_KEY: ${{ secrets.SAI_SSH_PRIVATE_KEY }} + run: | + set -euo pipefail + bash ci/sai/configure_ssh_client.sh \ + "$RUNNER_TEMP/sai-ssh" "$GITHUB_WORKSPACE/ci/sai/known_hosts" + echo "SAI_SSH_CONFIG=$RUNNER_TEMP/sai-ssh/config" >> "$GITHUB_ENV" + + - name: Run read-only SAI probe + run: | + set -euo pipefail + connected=0 + for attempt in {1..4}; do + if ssh -F "$SAI_SSH_CONFIG" -o ConnectionAttempts=1 -MNf sai-ci; then + connected=1 + break + fi + echo "SAI SSH master connection failed (attempt $attempt/4)" >&2 + if [[ $attempt -lt 4 ]]; then + sleep $((attempt * 5)) + fi + done + [[ $connected -eq 1 ]] + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s < ci/sai/probe_remote_sai.sh + + - name: Install cleanup cron + if: inputs.mode == 'install-cleanup' + env: + PROJECT_ROOT_INPUT: ${{ inputs.project_root }} + run: | + set -euo pipefail + project_root=${PROJECT_ROOT_INPUT:-$SAI_PROJECT_ROOT_DEFAULT} + : "${project_root:?Set the SAI_PROJECT_ROOT Environment variable or workflow input}" + [[ "$project_root" =~ ^/home/abacus-group/abacususer01(/[A-Za-z0-9._-]+)+$ ]] + case "/$project_root/" in + */./*|*/../*) + echo "Project root contains a dot path component: $project_root" >&2 + exit 1 + ;; + esac + run_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + output=$(ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- \ + "$project_root" "$run_key" \ + < ci/sai/prepare_cleanup_install.sh) + printf '%s\n' "$output" + remote_project_root=$(awk -F= '$1 == "SAI_PROJECT_ROOT" {print $2}' <<< "$output") + staging_file=$(awk -F= '$1 == "CLEANUP_STAGING_FILE" {print $2}' <<< "$output") + [[ "$remote_project_root" =~ ^/home/abacus-group/abacususer01/ ]] + [[ "$staging_file" == "$remote_project_root/diagnostics/cleanup-$run_key/cleanup_sai_runs.sh" ]] + rsync -az -e "ssh -F $SAI_SSH_CONFIG" \ + ci/sai/cleanup_sai_runs.sh \ + "sai-ci:$staging_file" + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- \ + "$remote_project_root" "$staging_file" \ + < ci/sai/install_cleanup_cron.sh + + - name: Remove local SSH credentials + if: always() + run: | + if [[ -f ${SAI_SSH_CONFIG:-} ]]; then + ssh -F "$SAI_SSH_CONFIG" -O exit sai-ci 2>/dev/null || true + fi + rm -rf "$RUNNER_TEMP/sai-ssh" diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml new file mode 100644 index 00000000000..5c5c11ecc63 --- /dev/null +++ b/.github/workflows/sai-gpu-full.yml @@ -0,0 +1,323 @@ +name: SAI GPU Case Matrix + +on: + workflow_dispatch: + inputs: + source_sha: + description: "Approved code SHA; executes as abacususer01 on SAI" + required: true + type: string + project_root: + description: "Optional absolute project root below the SAI account HOME" + required: false + default: "" + type: string + run_namespace: + description: "Optional run subdirectory, for example pr-7658" + required: false + default: "manual" + type: string + schedule: + - cron: "30 20 * * *" + +permissions: + contents: read + +concurrency: + group: sai-gpu-${{ github.event_name == 'schedule' && 'daily' || github.run_id }} + cancel-in-progress: false + +jobs: + rebuild-and-test: + name: SAI SSH rebuild and GPU validation + runs-on: ubuntu-24.04 + timeout-minutes: 240 + environment: + name: ${{ github.event_name == 'schedule' && 'sai-ssh-scheduled' || 'sai-ssh-manual' }} + env: + SOURCE_SHA: ${{ github.event_name == 'schedule' && github.sha || inputs.source_sha }} + PROJECT_ROOT_INPUT: ${{ inputs.project_root || '' }} + RUN_NAMESPACE_INPUT: ${{ inputs.run_namespace || '' }} + SAI_PROJECT_ROOT_DEFAULT: ${{ vars.SAI_PROJECT_ROOT }} + SAI_SSH_HOST: ${{ vars.SAI_SSH_HOST }} + SAI_SSH_PORT: ${{ vars.SAI_SSH_PORT }} + SAI_SSH_USER: ${{ vars.SAI_SSH_USER }} + ARTIFACT_ROOT: "" + REMOTE_PROJECT_ROOT: "" + REMOTE_RUN_ROOT: "" + REMOTE_SOURCE_TRANSFER_ROOT: "" + SOURCE_CACHE_BASE_SHA: "" + SOURCE_MANIFEST: "" + SOURCE_PAYLOAD: "" + SOURCE_PAYLOAD_MODE: "" + SAI_SSH_CONFIG: "" + steps: + - name: Checkout trusted CI control plane + uses: actions/checkout@v6 + with: + ref: ${{ github.event.repository.default_branch }} + path: control + persist-credentials: false + + - name: Validate requested source + run: | + set -euo pipefail + [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] + requested_root=${PROJECT_ROOT_INPUT:-$SAI_PROJECT_ROOT_DEFAULT} + : "${requested_root:?Set the SAI_PROJECT_ROOT Environment variable or workflow input}" + [[ "$requested_root" =~ ^/home/abacus-group/abacususer01(/[A-Za-z0-9._-]+)+$ ]] + case "/$requested_root/" in + */./*|*/../*) + echo "Project root contains a dot path component: $requested_root" >&2 + exit 1 + ;; + esac + if [[ "$GITHUB_EVENT_NAME" == schedule ]]; then + run_namespace=daily + source_cache_role=baseline + else + run_namespace=${RUN_NAMESPACE_INPUT:-manual} + source_cache_role=candidate + fi + [[ "$run_namespace" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] + { + echo "REQUESTED_PROJECT_ROOT=$requested_root" + echo "RUN_NAMESPACE=$run_namespace" + echo "SOURCE_CACHE_ROLE=$source_cache_role" + } >> "$GITHUB_ENV" + + - name: Checkout requested ABACUS commit + uses: actions/checkout@v6 + with: + ref: ${{ env.SOURCE_SHA }} + path: source + persist-credentials: false + + - name: Verify immutable source checkout + run: | + set -euo pipefail + actual_sha=$(git -C source rev-parse HEAD) + [[ "$actual_sha" == "$SOURCE_SHA" ]] + control_sha=$(git -C control rev-parse HEAD) + git -C source diff --exit-code + git -C source diff --cached --exit-code + git -C control diff --exit-code + git -C control diff --cached --exit-code + echo "SOURCE_SHA=$actual_sha" + echo "CONTROL_SHA=$control_sha" >> "$GITHUB_ENV" + + - name: Configure pinned SSH client + env: + SAI_SSH_PRIVATE_KEY: ${{ secrets.SAI_SSH_PRIVATE_KEY }} + run: | + set -euo pipefail + bash control/ci/sai/configure_ssh_client.sh \ + "$RUNNER_TEMP/sai-ssh" "$GITHUB_WORKSPACE/control/ci/sai/known_hosts" + echo "SAI_SSH_CONFIG=$RUNNER_TEMP/sai-ssh/config" >> "$GITHUB_ENV" + + - name: Verify SAI SSH and Slurm access + run: | + set -euo pipefail + connected=0 + for attempt in {1..4}; do + if ssh -F "$SAI_SSH_CONFIG" -o ConnectionAttempts=1 -MNf sai-ci; then + connected=1 + break + fi + echo "SAI SSH master connection failed (attempt $attempt/4)" >&2 + if [[ $attempt -lt 4 ]]; then + sleep $((attempt * 5)) + fi + done + [[ $connected -eq 1 ]] + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s \ + < control/ci/sai/probe_remote_sai.sh + + - name: Create isolated remote run + run: | + set -euo pipefail + run_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + output=$(ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- \ + "$REQUESTED_PROJECT_ROOT" "$RUN_NAMESPACE" "$run_key" \ + "$SOURCE_SHA" "$CONTROL_SHA" \ + < control/ci/sai/prepare_remote_run.sh) + printf '%s\n' "$output" + remote_project_root=$(awk -F= '$1 == "SAI_PROJECT_ROOT" {print $2}' <<< "$output") + remote_run_root=$(awk -F= '$1 == "RUN_ROOT" {print $2}' <<< "$output") + [[ "$remote_project_root" =~ ^/home/abacus-group/abacususer01/ ]] + [[ "$remote_run_root" == "$remote_project_root/runs/$RUN_NAMESPACE/$run_key" ]] + { + echo "REMOTE_PROJECT_ROOT=$remote_project_root" + echo "REMOTE_RUN_ROOT=$remote_run_root" + } >> "$GITHUB_ENV" + + - name: Upload trusted control plane to SAI + run: | + set -euo pipefail + rsync -az --delete --timeout=600 --stats \ + -e "ssh -F $SAI_SSH_CONFIG" \ + "$GITHUB_WORKSPACE/control/ci/sai/" \ + "sai-ci:$REMOTE_RUN_ROOT/control/" + + - name: Prepare source delta target + run: | + set -euo pipefail + output=$(ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- \ + prepare "$REMOTE_PROJECT_ROOT" "$REMOTE_RUN_ROOT" "$SOURCE_SHA" \ + "$SOURCE_CACHE_ROLE" \ + < control/ci/sai/source_transfer_cache.sh) + printf '%s\n' "$output" + transfer_root=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + base_sha=$(awk -F= '$1 == "SOURCE_CACHE_BASE_SHA" {print $2}' <<< "$output") + cache_role=$(awk -F= '$1 == "SOURCE_CACHE_ROLE" {print $2}' <<< "$output") + run_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + [[ "$transfer_root" == "$REMOTE_PROJECT_ROOT/cache/source-transfers/$run_key" ]] + [[ "$base_sha" == none || "$base_sha" =~ ^[0-9a-f]{40}$ ]] + [[ "$cache_role" == "$SOURCE_CACHE_ROLE" ]] + { + echo "REMOTE_SOURCE_TRANSFER_ROOT=$transfer_root" + echo "SOURCE_CACHE_BASE_SHA=$base_sha" + } >> "$GITHUB_ENV" + + - name: Build compressed source payload + run: | + set -euo pipefail + payload="$RUNNER_TEMP/source-payload.gz" + manifest="$RUNNER_TEMP/source-manifest.gz" + git -C source ls-tree -r -z --full-tree "$SOURCE_SHA" \ + | gzip -1 > "$manifest" + mode=full + if [[ "$SOURCE_CACHE_BASE_SHA" != none ]]; then + if git -C source cat-file -e "$SOURCE_CACHE_BASE_SHA^{commit}" 2>/dev/null || \ + git -C source fetch --no-tags --depth=1 origin "$SOURCE_CACHE_BASE_SHA"; then + git -C source diff --binary --full-index --no-renames \ + "$SOURCE_CACHE_BASE_SHA" "$SOURCE_SHA" | gzip -1 > "$payload" + mode=delta + else + echo "Cached SHA is unavailable from GitHub; sending a full snapshot" + fi + fi + if [[ "$mode" == full ]]; then + empty_tree=$(git -C source hash-object -t tree /dev/null) + git -C source diff --binary --full-index --no-renames \ + "$empty_tree" "$SOURCE_SHA" | gzip -1 > "$payload" + fi + [[ -s "$payload" && -s "$manifest" ]] + { + echo "SOURCE_PAYLOAD=$payload" + echo "SOURCE_MANIFEST=$manifest" + echo "SOURCE_PAYLOAD_MODE=$mode" + } >> "$GITHUB_ENV" + echo "SOURCE_PAYLOAD_READY mode=$mode base=$SOURCE_CACHE_BASE_SHA target=$SOURCE_SHA bytes=$(stat -c %s "$payload")" + + - name: Upload and apply compressed source payload + run: | + set -euo pipefail + started=$(date +%s) + rsync -a --partial --timeout=600 --stats \ + -e "ssh -F $SAI_SSH_CONFIG" \ + "$SOURCE_PAYLOAD" "$SOURCE_MANIFEST" \ + "sai-ci:$REMOTE_SOURCE_TRANSFER_ROOT/" + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- \ + receive "$REMOTE_PROJECT_ROOT" "$REMOTE_RUN_ROOT" \ + "$REMOTE_SOURCE_TRANSFER_ROOT" "$SOURCE_PAYLOAD_MODE" "$SOURCE_SHA" \ + < control/ci/sai/source_transfer_cache.sh + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- \ + finalize "$REMOTE_PROJECT_ROOT" "$REMOTE_RUN_ROOT" \ + "$REMOTE_SOURCE_TRANSFER_ROOT" "$SOURCE_SHA" \ + < control/ci/sai/source_transfer_cache.sh + echo "SOURCE_TRANSFER_SECONDS=$(($(date +%s) - started))" + + - name: Build and run SAI GPU validations + id: remote_validation + continue-on-error: true + run: | + set -uo pipefail + set +e + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- \ + "$REMOTE_PROJECT_ROOT" "$REMOTE_RUN_ROOT" "$SOURCE_SHA" "$CONTROL_SHA" \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" \ + < control/ci/sai/run_remote_ci.sh \ + 2>&1 | tee "$RUNNER_TEMP/sai-remote-driver.log" + remote_rc=${PIPESTATUS[0]} + set -e + echo "exit_code=$remote_rc" >> "$GITHUB_OUTPUT" + exit "$remote_rc" + + - name: Collect remote artifacts + id: collect + if: always() + continue-on-error: true + run: | + set -euo pipefail + ARTIFACT_ROOT=${ARTIFACT_ROOT:-$RUNNER_TEMP/sai-artifacts} + export ARTIFACT_ROOT + mkdir -p "$ARTIFACT_ROOT" + echo "ARTIFACT_ROOT=$ARTIFACT_ROOT" >> "$GITHUB_ENV" + { + echo "event_name=$GITHUB_EVENT_NAME" + echo "source_sha=$SOURCE_SHA" + echo "control_sha=${CONTROL_SHA:-unavailable}" + echo "remote_run_root=${REMOTE_RUN_ROOT:-unavailable}" + } > "$ARTIFACT_ROOT/workflow-context.txt" + if [[ -f "$RUNNER_TEMP/sai-remote-driver.log" ]]; then + cp "$RUNNER_TEMP/sai-remote-driver.log" "$ARTIFACT_ROOT/" + fi + if [[ -z ${SAI_SSH_CONFIG:-} || ! -f $SAI_SSH_CONFIG || \ + -z ${REMOTE_RUN_ROOT:-} ]]; then + echo "Remote run was not created; no remote artifacts are available." \ + | tee "$ARTIFACT_ROOT/collection-error.log" >&2 + exit 1 + fi + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- "$REMOTE_RUN_ROOT" \ + < control/ci/sai/collect_remote_artifacts.sh \ + | tar -xzf - -C "$ARTIFACT_ROOT" + + - name: Publish GPU case summary + if: always() + run: | + set -euo pipefail + summary="$ARTIFACT_ROOT/results/case-matrix/gpu-case-summary.md" + if [[ -f "$summary" ]]; then + cat "$summary" >> "$GITHUB_STEP_SUMMARY" + else + echo "## SAI GPU validation produced no case summary" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload SAI logs + id: artifact_upload + if: always() + uses: actions/upload-artifact@v7 + with: + name: sai-ssh-gpu-rebuild-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.ARTIFACT_ROOT }} + if-no-files-found: warn + retention-days: 30 + + - name: Mark remote artifacts as uploaded + if: always() && steps.collect.outcome == 'success' && steps.artifact_upload.outcome == 'success' + run: | + set -euo pipefail + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- "$REMOTE_RUN_ROOT" \ + < control/ci/sai/mark_artifacts_uploaded.sh + + - name: Remove local SSH credentials + if: always() + run: | + if [[ -f ${SAI_SSH_CONFIG:-} ]]; then + ssh -F "$SAI_SSH_CONFIG" -O exit sai-ci 2>/dev/null || true + fi + rm -rf "$RUNNER_TEMP/sai-ssh" + + - name: Report validation result + if: always() + env: + REMOTE_OUTCOME: ${{ steps.remote_validation.outcome }} + COLLECT_OUTCOME: ${{ steps.collect.outcome }} + REMOTE_EXIT_CODE: ${{ steps.remote_validation.outputs.exit_code }} + run: | + set -euo pipefail + echo "Remote validation: $REMOTE_OUTCOME (rc=${REMOTE_EXIT_CODE:-missing})" + echo "Artifact collection: $COLLECT_OUTCOME" + [[ "$REMOTE_OUTCOME" == success ]] + [[ "$COLLECT_OUTCOME" == success ]] diff --git a/ci/sai/README.md b/ci/sai/README.md new file mode 100644 index 00000000000..3efb86b2d22 --- /dev/null +++ b/ci/sai/README.md @@ -0,0 +1,227 @@ +# SAI GPU validation through SSH + +`.github/workflows/sai-gpu-full.yml` runs on a GitHub-hosted +`ubuntu-24.04` runner. It checks out the trusted control plane from the +repository default branch, checks out the requested ABACUS source commit +separately, and connects to SAI over pinned-host-key SSH as `abacususer01`. +No self-hosted GitHub runner or inbound service on SAI is required. + +The control checkout's exact commit is recorded as `CONTROL_SHA`. Its +`ci/sai` directory is uploaded to the run's `control` directory, and all +Slurm submission, toolchain, matrix, and launcher scripts execute from that +copy. `SOURCE_SHA` identifies the ABACUS source and test data being built. +Building an approved source commit still executes that commit's build system +and code as `abacususer01`; the control-plane separation does not sandbox an +untrusted source commit. + +## Triggers and trust boundary + +The daily schedule is `30 20 * * *` (04:30 the following day in Beijing). +Scheduled runs test the default-branch commit supplied by GitHub and use the +`sai-ssh-scheduled` Environment. That Environment has no required reviewer and +allows deployments only from the default branch. + +Manual dispatch requires an exact 40-character source commit SHA and uses the +protected `sai-ssh-manual` Environment. Configure `Stardust0831` as its required +reviewer and leave self-approval enabled. The reviewer must inspect the commit +before approval because its code will execute on SAI with all permissions of +`abacususer01`. External contributors only submit pull requests; a maintainer +reviews the pull request and then dispatches its exact commit when GPU testing +is warranted. Do not add an automatic `pull_request` trigger. + +Different manually approved commits may run concurrently. Scheduled runs share +one `daily` concurrency group and therefore serialize with other scheduled +runs, while every manual dispatch uses its GitHub run ID as an independent +group. The protected Environment approval remains per run. + +### One-time GitHub configuration + +Configure these settings in the upstream repository after this workflow is +merged. Do not upload the SSH key to a fork or create a repository-level +secret. + +1. Open **Settings > Environments** and create `sai-ssh-scheduled`. +2. Restrict its deployment branches to the default branch and do not add a + required reviewer. +3. Add an Environment secret named `SAI_SSH_PRIVATE_KEY`. Its value is the + complete contents of `~/.ssh/abacususer01`, including the `BEGIN` and `END` + lines. GitHub accepts the key as pasted text, not as a file upload. +4. Add the Environment variables shown below. +5. Create `sai-ssh-manual`, restrict it to the default branch, and add the + designated maintainer as a required reviewer. Allow self-review if that + maintainer must be able to dispatch and approve the same run. +6. Add the same Environment secret and variables to `sai-ssh-manual`. GitHub + does not share secrets or variables between Environments. + +Both Environments use these variables: + +```text +SAI_SSH_HOST=c0.sai.ai-4s.com +SAI_SSH_PORT=12022 +SAI_SSH_USER=abacususer01 +SAI_PROJECT_ROOT=/home/abacus-group/abacususer01/agent/abacus_sai_gpu_ci +``` + +`SAI_PROJECT_ROOT` is a default, not a compiled-in path. A manual run may use +another absolute project root whose components contain only letters, digits, +dot, underscore, or hyphen. The remote setup resolves existing symlinks and +rejects any path that escapes the account's canonical HOME. Only +`abacususer01` is supported by this deployment. + +### Manual run + +1. Review the source commit that will execute as `abacususer01` and obtain its + full 40-character SHA. For a pull request, use its head commit, not a merge + ref. +2. Open **Actions > SAI GPU Case Matrix > Run workflow** and select the + repository default branch. The workflow control scripts always come from + that trusted branch. +3. Enter the reviewed SHA as `source_sha`. +4. Leave `project_root` empty to use the manual Environment's + `SAI_PROJECT_ROOT`, or enter another absolute directory below + `/home/abacus-group/abacususer01`, for example + `/home/abacus-group/abacususer01/agent/abacus_sai_gpu_ci_trial`. +5. Set `run_namespace` to a short label such as `pr-7658`. Runs using the same + project root share the daily source baseline and NVIDIA archive cache, but + keep build, install, and result files in separate namespace directories. +6. Submit the workflow. A required reviewer then opens the pending deployment, + checks the requested SHA and directory, and approves `sai-ssh-manual`. + +The selected directory is a reusable project root, not a checkout directory. +Each attempt uses a new +`runs//-` subdirectory. Official NVIDIA +archives are cached under `vendor/`, and the user-level cleanup service +discovers every selected project root through its registry. A scheduled run +has no input form, always uses namespace `daily`, and uses the scheduled +Environment's `SAI_PROJECT_ROOT` with the current default-branch SHA. Choosing +a different project root intentionally creates an independent source baseline +and vendor cache; use a namespace below the default project root when sharing +those caches is desired. + +The SSH client disables agent and port forwarding, uses `BatchMode`, requires +the repository-pinned SAI host key, and enables transport compression for all +control commands and rsync traffic. Source payloads and manifests are already +gzip-compressed before rsync starts, while returned artifacts are created as a +gzip-compressed tar stream on SAI. The read-only probe establishes one SSH +master connection with bounded connection retries; later SSH and rsync steps +reuse it instead of repeatedly negotiating with the login gateway. The +workflow explicitly closes the master before removing the private key and +socket directory in an `always()` step. The key is written only to the GitHub +runner's temporary directory with mode 0600. Never print the key or pass it on +a command line. This connection retry does not resume a Slurm coordinator +after a mid-session disconnect; the current signal handlers cancel recorded +jobs to avoid leaving orphan allocations. + +## Build and GPU jobs + +Each attempt creates a collision-resistant directory at +`$SAI_PROJECT_ROOT/runs/$RUN_NAMESPACE/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT`. +The project root also holds a shared, locked cache of the SHA256-pinned +official NVIDIA archives: + +```text +cuSolverMp 0.9.0.6427 (CUDA 12) +cuBLASMp 0.9.1.3056 (CUDA 12) +``` + +An extracted archive is reused only when its `.archive-sha256` marker equals +the pinned digest. Unexpected, symlinked, or mismatched cache directories are +rejected rather than overwritten. First extraction uses a temporary directory +and an atomic rename. + +The active `archive-mp09-sai-nccl2293` profile combines those NVIDIA archives +with SAI's NCCL 2.29.3 library and the NVHPC/Open MPI/CUDA 12.9 toolchain. It +loads `nvhpc/26.3-gnu-cuda12-tuned`, whose SAI modulefile exports the same +`NCCL_ROOT` as `nccl/2.29.3-sai-cuda12.9`. The two modules intentionally +conflict because the tuned NVHPC module already includes that SAI NCCL stack; +the workflow therefore must not load both. The SAI NCCL build derives from +NVIDIA NCCL 2.29.3 and adds the operator's dual-rail channel policy; it is not +an unmodified NVIDIA binary. +Runtime checks require the expected `libnccl.so.2`, +`NCCL_SAI_RAIL_BY_CHANNEL=1`, cuSolverMp 0.9.0, cuBLASMp 0.9.1, and NCCL +2.29.3. The workflow does not modify `/opt`, modules, or system configuration. + +Source transfer keeps one verified, non-executed daily baseline snapshot and +its commit SHA under the selected project root. For the first run, GitHub sends +a gzip-compressed full snapshot represented as a binary diff from Git's empty +tree. Later runs ask SAI for that baseline SHA and send the same compressed +diff format from the baseline to the requested commit. A canonical Git tree +manifest accompanies every payload. SAI checks every path, file type, +executable bit, symlink target and blob hash, and rejects extra filesystem +entries before use. An invalid or unavailable cache pointer falls back to a +full snapshot. A manual candidate leaves an invalid baseline untouched; the +next scheduled baseline run may quarantine and replace it. + +Only a scheduled `daily` run promotes its verified source to the next +baseline. A manual PR run consumes the baseline but never advances it, even +when its tests pass or fail, so concurrent PRs do not form an accidental +rolling cache chain. Cache preparation and promotion hold a shared filesystem +lock; each run applies its payload in a private transfer directory before +copying it into the isolated run. The pre-existing `source-latest` snapshot is +accepted as the initial daily baseline when this policy is first deployed. +Build, install, and result directories are never reused, and tested code never +executes from or writes into the source cache. + +The build disables DeePMD, Torch/DeepKS, PEXSI, DFT-D4, LibRI, NEP, and cnpy +because the selected GPU suites do not exercise them. After a successful +build, three resource-homogeneous Slurm arrays submit all 48 cases in suites +11/12/13/15/16 while a 2-node, 16-rank Si48 cuSolverMp RT-TDDFT smoke job runs in +parallel. The arrays use one 1-GPU task, seven 2-GPU tasks, and forty 4-GPU +tasks. The 1/2-GPU arrays use `flood-1o2gpu`; the 4-GPU array and multinode job +use `flood-gpu`. Their per-class concurrent task caps are 2, 8, and 8, +respectively. No job pins a node name or explicitly requests CPU resources. + +SAI's partition mapping scripts determine MPI placement and OpenMP threads. +The validation keeps InfiniBand enabled and records effective MPI, UCX, NCCL, +RDMA, dynamic-library, Slurm accounting, and per-case numerical results. +Existing numerical references and thresholds are not relaxed. The currently +known `16_SDFT_GPU/005_PW_SDFT_MALL_BPCG_GPU` numerical failure therefore +makes the full workflow red until ABACUS fixes it. A case is retried exactly +once only when its first attempt fails before MPI initialization with the +observed PMIx shared-memory startup signature. Both attempt logs and retry +metadata are retained. A second PMIx startup failure is reported as +infrastructure rather than as an ABACUS numerical failure. + +## Artifacts and cleanup + +The GitHub artifact is retained for 30 days. It includes source/control commit +IDs, build identity and linkage, complete coordinator and Slurm logs, terminal +accounting, the 48-row case summary, per-case status, and ABACUS numerical +outputs. Collection uses an explicit extension/name allowlist. A failed setup +still uploads local workflow context instead of hiding the first error behind +an undefined path. + +After GitHub confirms artifact upload, the remote run gets an atomic +`.artifacts-uploaded` marker. `.github/workflows/sai-bootstrap.yml` has a +read-only `probe` mode and an `install-cleanup` mode. Both are manual and use +the protected manual Environment. Installation copies the cleanup program to +`$HOME/.local/libexec/abacus-sai-ci` and installs this user crontab entry: + +```text +15 7 * * * $HOME/.local/libexec/abacus-sai-ci/cleanup_sai_runs.sh --cron +``` + +Cleanup deletes uploaded runs 72 hours after their upload marker. Incomplete +runs and directories explicitly marked `.ci-diagnostic` are deleted after 168 +hours. Bootstrap stages the cleanup program in one of these marked diagnostic +directories and removes it after successful installation, so an interrupted +install follows the same retention rule. It discovers relocatable project +roots through a locked per-user registry, refuses roots outside HOME, and skips +runs whose recorded Slurm jobs are active. Failure to query a recorded job is +treated as unknown and also skipped. `--dry-run` prints the decisions without +deleting anything. Cleanup logs are stored under +`$HOME/.local/state/abacus-sai-ci` and rotate at 10 MiB. + +## Local verification + +The shell regression suite uses only temporary directories and mocked Slurm +commands: + +```bash +bash ci/sai/tests/test_sai_ci.sh +``` + +It covers SSH configuration, project-root containment and collision rejection, +archive-cache reuse and mismatch rejection, empty/populated artifact +collection, cleanup retention and job-query safety, and TERM/HUP cancellation +including the Slurm submission launch window. diff --git a/ci/sai/build_gpu.sbatch b/ci/sai/build_gpu.sbatch new file mode 100755 index 00000000000..d50e9217fcd --- /dev/null +++ b/ci/sai/build_gpu.sbatch @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +#SBATCH --job-name=abacus-sai-build +#SBATCH --partition=16V100 +#SBATCH --qos=huge-gpu +#SBATCH --nodes=1 +#SBATCH --ntasks=4 +#SBATCH --gpus-per-node=4 +#SBATCH --time=01:00:00 + +set -euo pipefail + +: "${CI_SOURCE:?}" +: "${CONTROL_ROOT:?}" +bash "$CONTROL_ROOT/build_gpu.sh" diff --git a/ci/sai/build_gpu.sh b/ci/sai/build_gpu.sh new file mode 100755 index 00000000000..4ff77af5372 --- /dev/null +++ b/ci/sai/build_gpu.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${CI_SOURCE:?CI_SOURCE must point to the staged ABACUS checkout}" +: "${BUILD_ROOT:?BUILD_ROOT must be set}" +: "${INSTALL_ROOT:?INSTALL_ROOT must be set}" +: "${TOOLCHAIN_FILE:?TOOLCHAIN_FILE must be set}" + +source "$TOOLCHAIN_FILE" +declare -F sai_load_toolchain >/dev/null || { + echo "Missing sai_load_toolchain() in $TOOLCHAIN_FILE" >&2 + exit 1 +} +sai_load_toolchain + +: "${SAI_CUDA_ROOT:?}" +: "${SAI_NVHPC_ROOT:?}" +: "${SAI_CUSOLVERMP_ROOT:?}" +: "${SAI_CUBLASMP_ROOT:?}" +: "${SAI_NCCL_ROOT:?}" + +MPI_CC=$(command -v mpicc) +MPI_CXX=$(command -v mpicxx) +MPI_RUN=$(command -v mpirun) +SAI_MPI_ROOT=$(cd "$(dirname "$MPI_CC")/.." && pwd) +export SAI_MPI_ROOT + +for path in \ + "$CI_SOURCE/CMakeLists.txt" \ + "$SAI_CUDA_ROOT/bin/nvcc" \ + "$SAI_CUSOLVERMP_ROOT/include/cusolverMp.h" \ + "$SAI_CUSOLVERMP_ROOT/lib/libcusolverMp.so.0" \ + "$SAI_CUBLASMP_ROOT/include/cublasmp.h" \ + "$SAI_CUBLASMP_ROOT/lib/libcublasmp.so.0" \ + "$SAI_NCCL_ROOT/include/nccl.h" \ + "$SAI_NCCL_ROOT/lib/libnccl.so" \ + "$SAI_NCCL_ROOT/lib/libnccl.so.2" \ + "$SAI_CUDA_ROOT/version.json"; do + [[ -e "$path" ]] || { + echo "Required toolchain path is missing: $path" >&2 + exit 1 + } +done + +read_define() { + local header=$1 + local name=$2 + awk -v name="$name" '$1 == "#define" && $2 == name {print $3; exit}' "$header" +} + +cusolver_header="$SAI_CUSOLVERMP_ROOT/include/cusolverMp.h" +cublas_header="$SAI_CUBLASMP_ROOT/include/cublasmp.h" +nccl_header="$SAI_NCCL_ROOT/include/nccl.h" +[[ $(read_define "$cusolver_header" CUSOLVERMP_VER_MAJOR) == 0 ]] +[[ $(read_define "$cusolver_header" CUSOLVERMP_VER_MINOR) == 9 ]] +[[ $(read_define "$cusolver_header" CUSOLVERMP_VER_PATCH) == 0 ]] +[[ $(read_define "$cublas_header" CUBLASMP_VER_MAJOR) == 0 ]] +[[ $(read_define "$cublas_header" CUBLASMP_VER_MINOR) == 9 ]] +[[ $(read_define "$cublas_header" CUBLASMP_VER_PATCH) == 1 ]] +[[ $(read_define "$nccl_header" NCCL_MAJOR) == 2 ]] +[[ $(read_define "$nccl_header" NCCL_MINOR) == 29 ]] +[[ $(read_define "$nccl_header" NCCL_PATCH) == 3 ]] +NCCL_LINK_LIBRARY=$(readlink -f "$SAI_NCCL_ROOT/lib/libnccl.so") +NCCL_RUNTIME_LIBRARY=$(readlink -f "$SAI_NCCL_ROOT/lib/libnccl.so.2") +[[ "$NCCL_LINK_LIBRARY" == "$NCCL_RUNTIME_LIBRARY" ]] +readelf --wide -Ws "$NCCL_LINK_LIBRARY" \ + | awk '$8 == "ncclCommQueryProperties" {found=1} END {exit !found}' +MPI_VERSION_OUTPUT=$("$MPI_RUN" --version) +CUDA_VERSION_OUTPUT=$("$SAI_CUDA_ROOT/bin/nvcc" --version) +grep -F 'Open MPI) 5.0.10' <<< "$MPI_VERSION_OUTPUT" >/dev/null +grep -F 'release 12.9, V12.9.86' <<< "$CUDA_VERSION_OUTPUT" >/dev/null +grep -F '"version" : "12.9.20250531"' "$SAI_CUDA_ROOT/version.json" >/dev/null + +mkdir -p "$BUILD_ROOT" "$INSTALL_ROOT" +export NVHPC_ROOT_DIR="$SAI_NVHPC_ROOT" +export CUSOLVERMP_PATH="$SAI_CUSOLVERMP_ROOT" +export CUBLASMP_PATH="$SAI_CUBLASMP_ROOT" +export NCCL_PATH="$SAI_NCCL_ROOT" +export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" + +cat > "$BUILD_ROOT/runtime-version-probe.c" <<'EOF' +#include +#include +#include + +typedef int (*get_version_t)(int *); + +static void *open_library(const char *path) +{ + void *handle = dlopen(path, RTLD_NOW | RTLD_LOCAL); + if (handle == NULL) { + fprintf(stderr, "dlopen(%s): %s\n", path, dlerror()); + exit(2); + } + return handle; +} + +int main(int argc, char **argv) +{ + int cublasmp_version = 0; + int nccl_version = 0; + void *cusolvermp; + void *cublasmp; + void *nccl; + get_version_t cublasmp_get_version; + get_version_t nccl_get_version; + + if (argc != 4) return 2; + cusolvermp = open_library(argv[1]); + cublasmp = open_library(argv[2]); + nccl = open_library(argv[3]); + if (dlsym(cusolvermp, "cusolverMpGetVersion") == NULL) return 3; + cublasmp_get_version = (get_version_t)dlsym(cublasmp, "cublasMpGetVersion"); + nccl_get_version = (get_version_t)dlsym(nccl, "ncclGetVersion"); + if (cublasmp_get_version == NULL || nccl_get_version == NULL) return 3; + if (cublasmp_get_version(&cublasmp_version) != 0) return 4; + if (nccl_get_version(&nccl_version) != 0) return 4; + printf("CUBLASMP_RUNTIME_VERSION=%d\n", cublasmp_version); + printf("NCCL_RUNTIME_VERSION=%d\n", nccl_version); + return cublasmp_version == 901 && nccl_version == 22903 ? 0 : 5; +} +EOF +"$MPI_CC" "$BUILD_ROOT/runtime-version-probe.c" -ldl \ + -o "$BUILD_ROOT/runtime-version-probe" +"$BUILD_ROOT/runtime-version-probe" \ + "$SAI_CUSOLVERMP_ROOT/lib/libcusolverMp.so.0" \ + "$SAI_CUBLASMP_ROOT/lib/libcublasmp.so.0" \ + "$NCCL_RUNTIME_LIBRARY" | tee "$BUILD_ROOT/runtime-version.txt" + +summary="$BUILD_ROOT/toolchain-summary.txt" +{ + echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" + echo "MPI_ROOT=$SAI_MPI_ROOT" + echo "MPI_RUN=$MPI_RUN" + echo "CUDA_ROOT=$SAI_CUDA_ROOT" + echo "CUSOLVERMP_ROOT=$SAI_CUSOLVERMP_ROOT" + echo "CUBLASMP_ROOT=$SAI_CUBLASMP_ROOT" + echo "NCCL_ROOT=$SAI_NCCL_ROOT" + echo "NCCL_LIBRARY=$NCCL_LINK_LIBRARY" + grep -E '^#define CUSOLVERMP_(VER_MAJOR|VER_MINOR|VER_PATCH|VERSION)' "$SAI_CUSOLVERMP_ROOT/include/cusolverMp.h" + grep -E '^#define CUBLASMP_(VER_MAJOR|VER_MINOR|VER_PATCH|VERSION)' "$SAI_CUBLASMP_ROOT/include/cublasmp.h" + grep -E '^#define NCCL_(MAJOR|MINOR|PATCH|VERSION_CODE)' "$SAI_NCCL_ROOT/include/nccl.h" + cat "$BUILD_ROOT/runtime-version.txt" + "$MPI_RUN" --version | sed -n '1,2p' + "$SAI_CUDA_ROOT/bin/nvcc" --version + module -t list 2>&1 +} | tee "$summary" + +cmake -S "$CI_SOURCE" -B "$BUILD_ROOT" -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_ROOT" \ + -DCMAKE_C_COMPILER="$MPI_CC" \ + -DCMAKE_CXX_COMPILER="$MPI_CXX" \ + -DCMAKE_CUDA_COMPILER="$SAI_CUDA_ROOT/bin/nvcc" \ + -DCMAKE_CUDA_ARCHITECTURES=70 \ + -DCMAKE_CUDA_FLAGS="-I$SAI_CUSOLVERMP_ROOT/include -I$SAI_CUBLASMP_ROOT/include" \ + -DCMAKE_CXX_FLAGS_RELEASE="-O3 -DNDEBUG -march=x86-64-v3 -mtune=generic" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,-rpath-link,$SAI_MPI_ROOT/lib" \ + -DENABLE_NATIVE_OPTIMIZATION=OFF \ + -DENABLE_MPI=ON -DENABLE_OPENMP=ON \ + -DUSE_CUDA=ON -DUSE_CUDA_MPI=ON \ + -DENABLE_LCAO=ON -DENABLE_ELPA=ON -DENABLE_LIBXC=ON \ + -DENABLE_LIBRI=OFF -DENABLE_MLALGO=OFF -DENABLE_RAPIDJSON=OFF \ + -DENABLE_PEXSI=OFF -DENABLE_DFTD4=OFF -DENABLE_CNPY=OFF \ + -DENABLE_CUSOLVERMP=ON -DENABLE_CUBLASMP=ON \ + -DENABLE_NCCL_PARALLEL_DEVICE=ON \ + -DBUILD_TESTING=OFF -DGIT_SUBMODULE=OFF \ + -DCAL_CUSOLVERMP_PATH="$SAI_CUSOLVERMP_ROOT" \ + -DCUSOLVERMP_LIBRARY="$SAI_CUSOLVERMP_ROOT/lib/libcusolverMp.so.0" \ + -DCUSOLVERMP_INCLUDE_DIR="$SAI_CUSOLVERMP_ROOT/include" \ + -DCUBLASMP_PATH="$SAI_CUBLASMP_ROOT" \ + -DCUBLASMP_LIBRARY="$SAI_CUBLASMP_ROOT/lib/libcublasmp.so.0" \ + -DCUBLASMP_INCLUDE_DIR="$SAI_CUBLASMP_ROOT/include" \ + -DNVHPC_ROOT_DIR="$SAI_NVHPC_ROOT" \ + -DNCCL_PATH="$SAI_NCCL_ROOT" \ + -DNCCL_LIBRARY="$SAI_NCCL_ROOT/lib/libnccl.so" \ + -DNCCL_INCLUDE_DIR="$SAI_NCCL_ROOT/include" + +cmake --build "$BUILD_ROOT" --parallel "${SAI_BUILD_JOBS:-32}" +cmake --install "$BUILD_ROOT" + +if [[ -x "$INSTALL_ROOT/bin/abacus_max_gpu" ]]; then + ln -sfn abacus_max_gpu "$INSTALL_ROOT/bin/abacus" +fi +ABACUS_BIN="$INSTALL_ROOT/bin/abacus" +[[ -x "$ABACUS_BIN" ]] || { + echo "ABACUS binary was not installed at $ABACUS_BIN" >&2 + exit 1 +} + +"$ABACUS_BIN" --info | tee "$INSTALL_ROOT/abacus-info.txt" +ldd "$ABACUS_BIN" | tee "$INSTALL_ROOT/ldd.txt" +if grep -q "not found" "$INSTALL_ROOT/ldd.txt"; then + echo "ABACUS has unresolved runtime dependencies" >&2 + exit 1 +fi +grep -q "$SAI_CUSOLVERMP_ROOT" "$INSTALL_ROOT/ldd.txt" +grep -q "$SAI_CUBLASMP_ROOT" "$INSTALL_ROOT/ldd.txt" +NCCL_LOADED_PATH=$(awk '$1 == "libnccl.so.2" {print $3; exit}' "$INSTALL_ROOT/ldd.txt") +[[ -n "$NCCL_LOADED_PATH" ]] +[[ $(readlink -f "$NCCL_LOADED_PATH") == "$NCCL_RUNTIME_LIBRARY" ]] +sha256sum "$ABACUS_BIN" | tee "$INSTALL_ROOT/abacus.sha256" + +echo "ABACUS_BUILD_PASSED binary=$ABACUS_BIN" diff --git a/ci/sai/cleanup_sai_runs.sh b/ci/sai/cleanup_sai_runs.sh new file mode 100755 index 00000000000..30adbc7cd86 --- /dev/null +++ b/ci/sai/cleanup_sai_runs.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +set -euo pipefail + +mode=normal +case ${1:-} in + '') ;; + --dry-run) mode=dry-run ;; + --cron) mode=cron ;; + *) echo "Usage: $0 [--dry-run|--cron]" >&2; exit 2 ;; +esac + +canonical_home=$(cd "$HOME" && pwd -P) +state_root=$(realpath --canonicalize-missing "$HOME/.local/state/abacus-sai-ci") +cache_root=$(realpath --canonicalize-missing "$HOME/.cache/abacus-sai-ci") +[[ $state_root == "$canonical_home/"* ]] +[[ $cache_root == "$canonical_home/"* ]] +mkdir -p "$state_root" "$cache_root" +state_root=$(cd "$state_root" && pwd -P) +cache_root=$(cd "$cache_root" && pwd -P) +[[ $state_root == "$canonical_home/"* ]] +[[ $cache_root == "$canonical_home/"* ]] +if [[ $mode == cron ]]; then + log="$state_root/cleanup.log" + if [[ -L $log || ( -e $log && ! -f $log ) || \ + -L $log.1 || ( -e $log.1 && ! -f $log.1 ) ]]; then + echo "Refusing unsafe cleanup log path under $state_root" >&2 + exit 1 + fi + if [[ -f $log && $(stat -c %s "$log") -gt 10485760 ]]; then + mv -T "$log" "$log.1" + fi + exec "$0" >> "$log" 2>&1 +fi + +exec 9<"$cache_root" +flock -n 9 || exit 0 +registry="$HOME/.config/abacus-sai-ci/project-roots" +[[ ! -L $registry ]] +[[ -f $registry ]] || exit 0 +now=$(date +%s) +squeue_checked=0 +squeue_ok=0 +active_job_ids= + +load_active_jobs() { + local output + [[ $squeue_checked -eq 0 ]] || return 0 + squeue_checked=1 + if ! output=$(squeue --noheader --user="$USER" --format='%F' 2>&1); then + echo "SQUEUE_QUERY_FAILED output=$output" + return 0 + fi + active_job_ids=$(awk '$1 ~ /^[0-9]+$/ {print $1}' <<< "$output" | sort -u) + squeue_ok=1 +} + +has_active_jobs() { + local run_root=$1 jobs job_id + jobs=$({ + if [[ -d $run_root/results ]]; then + find "$run_root/results" -type f -name '*submit.log' -exec \ + awk -F= '$1 == "SLURM_JOB_ID" && $2 ~ /^[0-9]+$/ {print $2}' {} + + find "$run_root/results" -type f -name array-jobs.tsv -exec \ + awk -F '\t' '$2 ~ /^[0-9]+$/ {print $2}' {} + + fi + } | sort -u | paste -sd, -) + [[ -n $jobs ]] || return 1 + load_active_jobs + if [[ $squeue_ok -ne 1 ]]; then + echo "SKIP squeue_failed run=$run_root jobs=$jobs" + return 0 + fi + while IFS= read -r job_id; do + if grep -Fxq "$job_id" <<< "$active_job_ids"; then + return 0 + fi + done < <(tr ',' '\n' <<< "$jobs") + return 1 +} + +remove_candidate() { + local path=$1 reason=$2 + if has_active_jobs "$path"; then + echo "SKIP active_or_unknown path=$path reason=$reason" + elif [[ $mode == dry-run ]]; then + echo "DRY_RUN delete path=$path reason=$reason" + else + rm -rf --one-file-system -- "$path" + echo "DELETED path=$path reason=$reason" + fi +} + +while IFS= read -r configured_root; do + [[ -n $configured_root ]] || continue + project_root=$(realpath --canonicalize-missing "$configured_root") + [[ $project_root == "$canonical_home/"* ]] || { + echo "SKIP root_outside_home path=$project_root" + continue + } + if [[ -d $project_root/runs ]]; then + while IFS= read -r -d '' run_root; do + run_name=${run_root##*/} + [[ $run_name =~ ^[0-9]+-[0-9]+$ ]] || continue + if [[ -f $run_root/.artifacts-uploaded ]]; then + stamp=$(stat -c %Y "$run_root/.artifacts-uploaded") + age_limit=259200 + reason=uploaded_over_72h + elif [[ -f $run_root/.ci-created ]]; then + stamp=$(stat -c %Y "$run_root/.ci-created") + age_limit=604800 + reason=incomplete_over_168h + else + continue + fi + (( now - stamp >= age_limit )) || continue + remove_candidate "$run_root" "$reason" + done < <(find "$project_root/runs" -mindepth 1 -maxdepth 2 -type d -print0) + fi + + if [[ -d $project_root/diagnostics ]]; then + while IFS= read -r -d '' diagnostic; do + [[ -f $diagnostic/.ci-diagnostic ]] || continue + stamp=$(stat -c %Y "$diagnostic/.ci-diagnostic") + (( now - stamp >= 604800 )) || continue + remove_candidate "$diagnostic" diagnostic_over_168h + done < <(find "$project_root/diagnostics" -mindepth 1 -maxdepth 1 -type d -print0) + fi + + transfer_root="$project_root/cache/source-transfers" + if [[ -d $transfer_root && ! -L $transfer_root ]]; then + transfer_root=$(realpath -e "$transfer_root") + [[ $transfer_root == "$project_root/cache/source-transfers" ]] + while IFS= read -r -d '' transfer; do + transfer_name=${transfer##*/} + [[ $transfer_name =~ ^[0-9]+-[0-9]+$ ]] || continue + marker="$transfer/.ci-source-transfer" + [[ -f $marker && ! -L $marker ]] || continue + stamp=$(stat -c %Y "$marker") + (( now - stamp >= 604800 )) || continue + remove_candidate "$transfer" source_transfer_over_168h + done < <(find "$transfer_root" -mindepth 1 -maxdepth 1 -type d -print0) + fi +done < "$registry" + +echo "CLEANUP_FINISHED mode=$mode at=$(date --iso-8601=seconds)" diff --git a/ci/sai/collect_remote_artifacts.sh b/ci/sai/collect_remote_artifacts.sh new file mode 100755 index 00000000000..679ab41516f --- /dev/null +++ b/ci/sai/collect_remote_artifacts.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 RUN_ROOT" >&2 + exit 2 +fi + +run_root=$(realpath -e "$1") +[[ $run_root == "$HOME/"*'/runs/'* ]] +cd "$run_root" + +list_file=$(mktemp "$run_root/.artifact-list.XXXXXX") +trap 'rm -f "$list_file"' EXIT +add_file() { + local canonical + [[ -e $1 || -L $1 ]] || return 0 + canonical=$(realpath -e "$1") + [[ $canonical == "$run_root/"* ]] + printf '%s\0' "$1" >> "$list_file" +} + +add_file .ci-created +add_file .artifacts-uploaded +for path in \ + build/toolchain-summary.txt build/CMakeCache.txt \ + install/abacus-info.txt install/ldd.txt install/abacus.sha256; do + add_file "$path" +done + +if [[ -d results ]]; then + find results -type f \( \ + -name '*.log' -o -name '*.out' -o -name '*.txt' -o \ + -name '*.tsv' -o -name '*.md' -o -name '*.sha256' \ + \) -print0 >> "$list_file" +fi +if [[ -d source/tests ]]; then + find source/tests -type f \( \ + -name log.txt -o -name result.out -o -name 'running*.log' -o \ + -name warning.log \ + \) -print0 >> "$list_file" +fi + +tar --no-recursion --null --files-from="$list_file" -czf - diff --git a/ci/sai/configure_ssh_client.sh b/ci/sai/configure_ssh_client.sh new file mode 100755 index 00000000000..c99a22cbb87 --- /dev/null +++ b/ci/sai/configure_ssh_client.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 OUTPUT_DIRECTORY KNOWN_HOSTS_FILE" >&2 + exit 2 +fi + +output_dir=$1 +known_hosts_source=$2 +: "${SAI_SSH_PRIVATE_KEY:?}" +: "${SAI_SSH_HOST:?}" +: "${SAI_SSH_PORT:?}" +: "${SAI_SSH_USER:?}" + +[[ $SAI_SSH_HOST =~ ^[A-Za-z0-9.-]+$ ]] +[[ $SAI_SSH_PORT =~ ^[0-9]+$ ]] +[[ $SAI_SSH_USER =~ ^[A-Za-z0-9._-]+$ ]] +[[ -f $known_hosts_source ]] +grep -Fq "[$SAI_SSH_HOST]:$SAI_SSH_PORT " "$known_hosts_source" + +umask 077 +mkdir -p "$output_dir" +key_file="$output_dir/id_ed25519" +known_hosts="$output_dir/known_hosts" +config="$output_dir/config" +printf '%s\n' "$SAI_SSH_PRIVATE_KEY" > "$key_file" +ssh-keygen -y -f "$key_file" >/dev/null +cp "$known_hosts_source" "$known_hosts" + +cat > "$config" <&2 + exit 2 +fi + +requested_root=$1 +requested_cleanup=$2 +canonical_home=$(cd "$HOME" && pwd -P) +project_root=$(realpath --canonicalize-missing "$requested_root") +[[ $project_root == "$canonical_home/"* ]] +mkdir -p "$project_root" +project_root=$(cd "$project_root" && pwd -P) +[[ $project_root == "$canonical_home/"* ]] + +staged_cleanup=$(realpath -e "$requested_cleanup") +[[ -f $staged_cleanup && ! -L $requested_cleanup ]] +staging_root=$(dirname "$staged_cleanup") +staging_name=${staging_root##*/} +[[ $staged_cleanup == "$staging_root/cleanup_sai_runs.sh" ]] +[[ $(dirname "$staging_root") == "$project_root/diagnostics" ]] +[[ $staging_name =~ ^cleanup-[0-9]+-[0-9]+$ ]] +[[ -f $staging_root/.ci-diagnostic ]] + +canonical_user_dir() { + local requested=$1 canonical + canonical=$(realpath --canonicalize-missing "$requested") + if [[ $canonical != "$canonical_home/"* ]]; then + echo "User state directory escapes HOME: $canonical" >&2 + return 1 + fi + mkdir -p "$canonical" + canonical=$(cd "$canonical" && pwd -P) + if [[ $canonical != "$canonical_home/"* ]]; then + echo "Created user state directory escapes HOME: $canonical" >&2 + return 1 + fi + printf '%s\n' "$canonical" +} + +config_root=$(canonical_user_dir "$HOME/.config/abacus-sai-ci") +state_root=$(canonical_user_dir "$HOME/.local/state/abacus-sai-ci") +libexec_root=$(canonical_user_dir "$HOME/.local/libexec/abacus-sai-ci") +: "$state_root" + +cleanup="$libexec_root/cleanup_sai_runs.sh" +registry="$config_root/project-roots" +exec 9<"$config_root" +flock 9 +if [[ -L $registry || ( -e $registry && ! -f $registry ) ]]; then + echo "Refusing unsafe project-root registry: $registry" >&2 + exit 1 +fi +if [[ -L $cleanup || ( -e $cleanup && ! -f $cleanup ) ]]; then + echo "Refusing unsafe cleanup installation target: $cleanup" >&2 + exit 1 +fi + +begin='# BEGIN ABACUS_SAI_CI_CLEANUP' +end='# END ABACUS_SAI_CI_CLEANUP' +current=$(mktemp "$state_root/cron-current.XXXXXX") +updated=$(mktemp "$state_root/cron-updated.XXXXXX") +crontab_error=$(mktemp "$state_root/cron-error.XXXXXX") +cleanup_tmp= +trap 'rm -f "$current" "$updated" "$crontab_error" ${cleanup_tmp:-}' EXIT +if ! LC_ALL=C crontab -l > "$current" 2> "$crontab_error"; then + if grep -Eq '^no crontab for ' "$crontab_error"; then + : > "$current" + else + echo "Unable to read the existing user crontab; refusing replacement" >&2 + cat "$crontab_error" >&2 + exit 1 + fi +fi +begin_count=$(grep -Fxc "$begin" "$current" || true) +end_count=$(grep -Fxc "$end" "$current" || true) +if [[ $begin_count -ne $end_count || $begin_count -gt 1 ]]; then + echo "Refusing to replace an invalid ABACUS SAI cleanup cron block" >&2 + exit 1 +fi +if [[ $begin_count -eq 1 ]]; then + begin_line=$(grep -Fn "$begin" "$current" | cut -d: -f1) + end_line=$(grep -Fn "$end" "$current" | cut -d: -f1) + if [[ $begin_line -ge $end_line ]]; then + echo "Refusing to replace an out-of-order ABACUS SAI cleanup cron block" >&2 + exit 1 + fi +fi +awk -v begin="$begin" -v end="$end" ' + $0 == begin {skip=1; next} + $0 == end {skip=0; next} + !skip {print} +' "$current" > "$updated" +{ + echo "$begin" + echo '15 7 * * * $HOME/.local/libexec/abacus-sai-ci/cleanup_sai_runs.sh --cron' + echo "$end" +} >> "$updated" + +cleanup_tmp=$(mktemp "$libexec_root/.cleanup_sai_runs.XXXXXX") +cp "$staged_cleanup" "$cleanup_tmp" +chmod 700 "$cleanup_tmp" +mv -T "$cleanup_tmp" "$cleanup" +cleanup_tmp= +touch "$registry" +if ! grep -Fxq "$project_root" "$registry"; then + printf '%s\n' "$project_root" >> "$registry" +fi +crontab "$updated" +rm -f "$staged_cleanup" "$staging_root/.ci-diagnostic" +rmdir "$staging_root" + +echo "SAI_CLEANUP_INSTALLED project_root=$project_root schedule=07:15" diff --git a/ci/sai/known_hosts b/ci/sai/known_hosts new file mode 100644 index 00000000000..52a041a2068 --- /dev/null +++ b/ci/sai/known_hosts @@ -0,0 +1 @@ +[c0.sai.ai-4s.com]:12022 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBurYQExmMdgat4VU1Twmu2pqxsBjLP6eff9oc6/8N3KUCtoe8HU9eONghETLECdA7A+O9DpefDRba0AcSXTfQc= diff --git a/ci/sai/mark_artifacts_uploaded.sh b/ci/sai/mark_artifacts_uploaded.sh new file mode 100755 index 00000000000..de25794696f --- /dev/null +++ b/ci/sai/mark_artifacts_uploaded.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 RUN_ROOT" >&2 + exit 2 +fi +run_root=$(realpath -e "$1") +[[ $run_root == "$HOME/"*'/runs/'* ]] +uploaded_tmp=$(mktemp "$run_root/.artifacts-uploaded.XXXXXX") +trap 'rm -f "$uploaded_tmp"' EXIT +{ + printf 'uploaded_epoch=%s\n' "$(date +%s)" + printf 'uploaded_iso=%s\n' "$(date --iso-8601=seconds)" +} > "$uploaded_tmp" +mv -T "$uploaded_tmp" "$run_root/.artifacts-uploaded" +trap - EXIT diff --git a/ci/sai/mpirun_with_mapping.sh b/ci/sai/mpirun_with_mapping.sh new file mode 100755 index 00000000000..42ff705cb41 --- /dev/null +++ b/ci/sai/mpirun_with_mapping.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${SAI_SYSTEM_MPIRUN:?}" +: "${MAP_OPT:?}" + +exec "$SAI_SYSTEM_MPIRUN" --map-by "$MAP_OPT" "$@" diff --git a/ci/sai/prepare_cleanup_install.sh b/ci/sai/prepare_cleanup_install.sh new file mode 100755 index 00000000000..187af782a7d --- /dev/null +++ b/ci/sai/prepare_cleanup_install.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 PROJECT_ROOT RUN_KEY" >&2 + exit 2 +fi + +requested_root=$1 +run_key=$2 +[[ $run_key =~ ^[0-9]+-[0-9]+$ ]] +[[ $requested_root =~ ^(/[A-Za-z0-9._-]+)+$ ]] +case "/$requested_root/" in + */./*|*/../*) + echo "Project root contains a dot path component: $requested_root" >&2 + exit 1 + ;; +esac + +canonical_home=$(cd "$HOME" && pwd -P) +project_root=$(realpath --canonicalize-missing "$requested_root") +[[ $project_root == "$canonical_home/"* ]] +mkdir -p "$project_root" +project_root=$(cd "$project_root" && pwd -P) +[[ $project_root == "$canonical_home/"* ]] + +config_root=$(realpath --canonicalize-missing "$HOME/.config/abacus-sai-ci") +[[ $config_root == "$canonical_home/"* ]] +mkdir -p "$config_root" +config_root=$(cd "$config_root" && pwd -P) +[[ $config_root == "$canonical_home/"* ]] +registry="$config_root/project-roots" +exec 9<"$config_root" +flock 9 +if [[ -L $registry || ( -e $registry && ! -f $registry ) ]]; then + echo "Refusing unsafe project-root registry: $registry" >&2 + exit 1 +fi +touch "$registry" +if ! grep -Fxq "$project_root" "$registry"; then + printf '%s\n' "$project_root" >> "$registry" +fi + +diagnostics_root=$(realpath --canonicalize-missing "$project_root/diagnostics") +[[ $diagnostics_root == "$project_root/diagnostics" ]] +mkdir -p "$diagnostics_root" +diagnostics_root=$(cd "$diagnostics_root" && pwd -P) +[[ $diagnostics_root == "$project_root/diagnostics" ]] + +staging_root="$diagnostics_root/cleanup-$run_key" +[[ ! -e $staging_root && ! -L $staging_root ]] +umask 077 +mkdir -p "$staging_root" +: > "$staging_root/.ci-diagnostic" +staging_file="$staging_root/cleanup_sai_runs.sh" +: > "$staging_file" + +printf 'SAI_PROJECT_ROOT=%s\n' "$project_root" +printf 'CLEANUP_STAGING_FILE=%s\n' "$staging_file" diff --git a/ci/sai/prepare_cusolvermp_smoke.sh b/ci/sai/prepare_cusolvermp_smoke.sh new file mode 100644 index 00000000000..585b05f7162 --- /dev/null +++ b/ci/sai/prepare_cusolvermp_smoke.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${CI_SOURCE:?}" +: "${RESULT_ROOT:?}" + +case_name=19_NO_Si48_CUSOLVERMP_TDDFT_GPU +source_case="$CI_SOURCE/tests/15_rtTDDFT_GPU/$case_name" +source_input="$source_case/INPUT" +smoke_root="$RESULT_ROOT/cusolvermp-smoke" +smoke_suite="$smoke_root/15_rtTDDFT_GPU" +smoke_case="$smoke_suite/$case_name" +smoke_input="$smoke_case/INPUT" +cusolvermp_pattern='^[[:space:]]*ks_solver[[:space:]]+cusolvermp[[:space:]]*$' +declare -A expected_sha=( + [INPUT]=1285180c32058368699e0ec8c0b50d73f6a8af3e89aabe2791c91a210ed57c54 + [KPT]=91042b39ee493cb5c1bee648adc865a2e5935a29278eb405dc009f994db55ece + [README]=19a018be686ce24a5e43684cac588e10cf72ade391fd08a6f600e316cfe48ce7 + [STRU]=8da05442b1f70f79b3decd603c94bd7d650b5f31df166db2916e981b61294760 +) + +[[ -d $source_case && ! -L $source_case ]] +for name in INPUT KPT README STRU; do + source_file=$source_case/$name + [[ -f $source_file && ! -L $source_file ]] + actual_sha=$(sha256sum "$source_file" | awk '{print $1}') + [[ $actual_sha == "${expected_sha[$name]}" ]] || { + echo "Unexpected $name hash in $source_case: $actual_sha" >&2 + exit 1 + } +done +[[ -d $CI_SOURCE/tests/PP_ORB ]] +[[ ! -e $smoke_root && ! -L $smoke_root ]] +[[ $(grep -Ec "$cusolvermp_pattern" "$source_input") -eq 1 ]] || { + echo "Expected exactly one ks_solver=cusolvermp line in $source_input" >&2 + exit 1 +} + +mkdir -p "$smoke_suite" +mkdir "$smoke_case" +for name in INPUT KPT README STRU; do + install -m 0644 -- "$source_case/$name" "$smoke_case/$name" +done +ln -s "$CI_SOURCE/tests/PP_ORB" "$smoke_root/PP_ORB" + +[[ $(grep -Ec "$cusolvermp_pattern" "$smoke_input") -eq 1 ]] +printf 'CUSOLVERMP_SMOKE_CASE=%s\n' "$smoke_case" diff --git a/ci/sai/prepare_gpu_case_matrix.sh b/ci/sai/prepare_gpu_case_matrix.sh new file mode 100755 index 00000000000..825764c0390 --- /dev/null +++ b/ci/sai/prepare_gpu_case_matrix.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${CI_SOURCE:?}" +: "${RESULT_ROOT:?}" + +matrix_root="$RESULT_ROOT/case-matrix" +manifest_root="$matrix_root/manifests" +[[ ! -e "$manifest_root" ]] || { + echo "Refusing to replace existing manifests: $manifest_root" >&2 + exit 1 +} +mkdir -p "$manifest_root" + +declare -A seen=() +declare -A counts=([gpu1]=0 [gpu2]=0 [gpu4]=0) + +add_case() { + local class=$1 + local suite=$2 + local case_name=$3 + local key="$suite/$case_name" + local case_dir="$CI_SOURCE/tests/$key" + + [[ $class =~ ^gpu(1|2|4)$ ]] + [[ $suite =~ ^[A-Za-z0-9_.-]+$ ]] + [[ $case_name =~ ^[A-Za-z0-9_.-]+$ ]] + [[ -d "$case_dir" ]] || { + echo "GPU case directory is missing: $case_dir" >&2 + exit 1 + } + [[ -f "$case_dir/result.ref" ]] || { + echo "GPU case has no numerical reference: $key" >&2 + exit 1 + } + [[ -z ${seen[$key]+x} ]] || { + echo "GPU case is assigned more than once: $key" >&2 + exit 1 + } + + seen[$key]=$class + counts[$class]=$((counts[$class] + 1)) + printf '%s\t%s\n' "$suite" "$case_name" >> "$manifest_root/$class.tsv" +} + +read_cases() { + local class=$1 + local suite=$2 + local cases_file="$CI_SOURCE/tests/$suite/CASES_GPU.txt" + local line case_name + + [[ -f "$cases_file" ]] + while IFS= read -r line || [[ -n $line ]]; do + case_name=${line%%#*} + read -r case_name <<< "$case_name" + [[ -n $case_name ]] || continue + add_case "$class" "$suite" "$case_name" + done < "$cases_file" +} + +while IFS= read -r case_name || [[ -n $case_name ]]; do + case_name=${case_name%%#*} + read -r case_name <<< "$case_name" + [[ -n $case_name ]] || continue + case "$case_name" in + scf_out_wf) add_case gpu1 11_PW_GPU "$case_name" ;; + scf_bpcg) add_case gpu2 11_PW_GPU "$case_name" ;; + *) add_case gpu4 11_PW_GPU "$case_name" ;; + esac +done < "$CI_SOURCE/tests/11_PW_GPU/CASES_GPU.txt" + +read_cases gpu4 12_NAO_Gamma_GPU +read_cases gpu4 13_NAO_multik_GPU +read_cases gpu4 15_rtTDDFT_GPU +read_cases gpu2 16_SDFT_GPU + +[[ ${counts[gpu1]} -eq 1 ]] +[[ ${counts[gpu2]} -eq 7 ]] +[[ ${counts[gpu4]} -eq 40 ]] +[[ ${#seen[@]} -eq 48 ]] + +summary="$matrix_root/inventory.txt" +{ + echo "GPU_CASE_MATRIX_TOTAL=${#seen[@]}" + for class in gpu1 gpu2 gpu4; do + echo "GPU_CASE_MATRIX_${class^^}=${counts[$class]}" + done +} | tee "$summary" + +echo "SAI_GPU_CASE_MANIFESTS_READY root=$manifest_root" diff --git a/ci/sai/prepare_nvidia_mp.sh b/ci/sai/prepare_nvidia_mp.sh new file mode 100755 index 00000000000..83457f7feaf --- /dev/null +++ b/ci/sai/prepare_nvidia_mp.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${SAI_PROJECT_ROOT:?SAI_PROJECT_ROOT must be set}" + +project_root=$(realpath -e "$SAI_PROJECT_ROOT") +vendor_root=${SAI_NVIDIA_MP_ROOT:-$project_root/vendor/nvidia-mp-0.9-archive} +vendor_root=$(realpath --canonicalize-missing "$vendor_root") +download_root=$(realpath --canonicalize-missing "$project_root/vendor/downloads") + +cusolver_archive=libcusolvermp-linux-x86_64-0.9.0.6427_cuda12-archive +cublas_archive=libcublasmp-linux-x86_64-0.9.1.3056_cuda12-archive +cusolver_tar=${cusolver_archive}.tar.xz +cublas_tar=${cublas_archive}.tar.xz +cusolver_sha=3b071ce69c6a6a6bb7add8784e6a3fc54e9a64a8f2c1c7da40b03bcde39eb57c +cublas_sha=35fea4df2bb08a496981f34c0d486f0753d3766a31d60dbe6daa6f16673cd1cc + +[[ "$project_root" == "$HOME/"* ]] +[[ "$vendor_root" == "$project_root/"* ]] +[[ "$download_root" == "$project_root/"* ]] +mkdir -p "$vendor_root" "$download_root" +vendor_root=$(cd "$vendor_root" && pwd -P) +download_root=$(cd "$download_root" && pwd -P) +[[ "$vendor_root" == "$project_root/"* ]] +[[ "$download_root" == "$project_root/"* ]] +exec 9<"$download_root" +flock 9 +temporary_paths=() +cleanup() { + local path + for path in "${temporary_paths[@]}"; do + case $path in + "$download_root"/*|"$vendor_root"/*) rm -rf -- "$path" ;; + esac + done +} +trap cleanup EXIT + +fetch_archive() { + local product=$1 + local package=$2 + local archive=$3 + local expected_sha=$4 + local tarball=$5 + local destination=$6 + + local temporary_tarball= + local temporary_destination= + + [[ "$destination" == "$vendor_root/"* ]] + + if [[ -e "$destination" || -L "$destination" ]]; then + if [[ ! -d "$destination" || -L "$destination" || \ + ! -f "$destination/.archive-sha256" || \ + $(<"$destination/.archive-sha256") != "$expected_sha" ]]; then + echo "Refusing unexpected or mismatched NVIDIA archive cache: $destination" >&2 + exit 1 + fi + echo "Reusing verified NVIDIA archive cache: $destination" + return + fi + + if [[ -L "$tarball" || ( -e "$tarball" && ! -f "$tarball" ) ]]; then + echo "Refusing unsafe NVIDIA archive tarball cache: $tarball" >&2 + exit 1 + fi + + if [[ -f "$tarball" ]] && ! printf '%s %s\n' "$expected_sha" "$tarball" | sha256sum --check --status; then + rm -f "$tarball" + fi + if [[ ! -f "$tarball" ]]; then + temporary_tarball=$(mktemp "$download_root/.${archive}.download.XXXXXX") + temporary_paths+=("$temporary_tarball") + curl --fail --location --retry 3 --connect-timeout 20 \ + --max-time 1200 \ + --output "$temporary_tarball" \ + "https://developer.download.nvidia.com/compute/${product}/redist/${package}/linux-x86_64/${archive}.tar.xz" + printf '%s %s\n' "$expected_sha" "$temporary_tarball" | sha256sum --check - + mv -T "$temporary_tarball" "$tarball" + fi + printf '%s %s\n' "$expected_sha" "$tarball" | sha256sum --check - + temporary_destination=$(mktemp -d "$vendor_root/.${archive}.extract.XXXXXX") + temporary_paths+=("$temporary_destination") + tar -xJf "$tarball" -C "$temporary_destination" --strip-components=1 + printf '%s\n' "$expected_sha" > "$temporary_destination/.archive-sha256" + mv -T "$temporary_destination" "$destination" +} + +fetch_archive cusolvermp libcusolvermp "$cusolver_archive" "$cusolver_sha" \ + "$download_root/$cusolver_tar" "$vendor_root/$cusolver_archive" +fetch_archive cublasmp libcublasmp "$cublas_archive" "$cublas_sha" \ + "$download_root/$cublas_tar" "$vendor_root/$cublas_archive" + +for path in \ + "$vendor_root/$cusolver_archive/include/cusolverMp.h" \ + "$vendor_root/$cusolver_archive/lib/libcusolverMp.so.0" \ + "$vendor_root/$cublas_archive/include/cublasmp.h" \ + "$vendor_root/$cublas_archive/lib/libcublasmp.so.0"; do + [[ -e "$path" ]] || { + echo "NVIDIA archive is incomplete: $path" >&2 + exit 1 + } +done +trap - EXIT + +if [[ -n ${GITHUB_ENV:-} ]]; then + { + echo "SAI_CUSOLVERMP_ROOT=$vendor_root/$cusolver_archive" + echo "SAI_CUBLASMP_ROOT=$vendor_root/$cublas_archive" + } >> "$GITHUB_ENV" +fi + +echo "NVIDIA_MP_ARCHIVES_READY root=$vendor_root" diff --git a/ci/sai/prepare_remote_run.sh b/ci/sai/prepare_remote_run.sh new file mode 100755 index 00000000000..4648d81c47d --- /dev/null +++ b/ci/sai/prepare_remote_run.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 5 ]]; then + echo "Usage: $0 PROJECT_ROOT RUN_NAMESPACE RUN_KEY SOURCE_SHA CONTROL_SHA" >&2 + exit 2 +fi + +requested_root=$1 +run_namespace=$2 +run_key=$3 +source_sha=$4 +control_sha=$5 +[[ $run_namespace =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] +[[ $run_key =~ ^[0-9]+-[0-9]+$ ]] +[[ $source_sha =~ ^[0-9a-fA-F]{40}$ ]] +[[ $control_sha =~ ^[0-9a-fA-F]{40}$ ]] +[[ $requested_root =~ ^(/[A-Za-z0-9._-]+)+$ ]] +case "/$requested_root/" in + */./*|*/../*) + echo "Project root contains a dot path component: $requested_root" >&2 + exit 1 + ;; +esac + +canonical_home=$(cd "$HOME" && pwd -P) +project_root=$(realpath --canonicalize-missing "$requested_root") +case $project_root in + "$canonical_home"/*) ;; + *) echo "Project root escapes HOME: $project_root" >&2; exit 1 ;; +esac + +config_root=$(realpath --canonicalize-missing "$HOME/.config/abacus-sai-ci") +[[ $config_root == "$canonical_home/"* ]] +runs_root=$(realpath --canonicalize-missing "$project_root/runs") +namespace_root=$(realpath --canonicalize-missing \ + "$runs_root/$run_namespace") +[[ $runs_root == "$project_root/runs" ]] +[[ $namespace_root == "$runs_root/$run_namespace" ]] +mkdir -p "$namespace_root" "$config_root" +project_root=$(cd "$project_root" && pwd -P) +runs_root=$(cd "$runs_root" && pwd -P) +namespace_root=$(cd "$namespace_root" && pwd -P) +config_root=$(cd "$config_root" && pwd -P) +[[ $project_root == "$canonical_home/"* ]] +[[ $runs_root == "$project_root/runs" ]] +[[ $namespace_root == "$runs_root/$run_namespace" ]] +[[ $config_root == "$canonical_home/"* ]] +run_root="$namespace_root/$run_key" +[[ ! -e $run_root && ! -L $run_root ]] || { + echo "Refusing to reuse remote run: $run_root" >&2 + exit 1 +} +mkdir -p "$run_root/source" "$run_root/control" "$run_root/build" \ + "$run_root/install" "$run_root/results" +{ + printf 'created_epoch=%s\n' "$(date +%s)" + printf 'created_iso=%s\n' "$(date --iso-8601=seconds)" + printf 'source_sha=%s\n' "$source_sha" + printf 'control_sha=%s\n' "$control_sha" +} > "$run_root/.ci-created" + +registry="$config_root/project-roots" +exec 9<"$config_root" +flock 9 +if [[ -L $registry || ( -e $registry && ! -f $registry ) ]]; then + echo "Refusing unsafe project-root registry: $registry" >&2 + exit 1 +fi +touch "$registry" +if ! grep -Fxq "$project_root" "$registry"; then + printf '%s\n' "$project_root" >> "$registry" +fi + +printf 'SAI_PROJECT_ROOT=%s\n' "$project_root" +printf 'RUN_ROOT=%s\n' "$run_root" diff --git a/ci/sai/probe_remote_sai.sh b/ci/sai/probe_remote_sai.sh new file mode 100755 index 00000000000..84657163b3d --- /dev/null +++ b/ci/sai/probe_remote_sai.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[[ $USER == abacususer01 ]] +[[ $HOME == /home/abacus-group/abacususer01 ]] +[[ $(id -u) -eq 1478400356 ]] +for command_name in sbatch sacct squeue scancel rsync curl git gzip tar xz \ + realpath flock crontab; do + command -v "$command_name" >/dev/null +done +sinfo -h -p 16V100 -o '%P %a %D %G' | grep -q '^16V100 ' +echo "SAI_SSH_PROBE_OK user=$USER home=$HOME host=$(hostname)" +sinfo -h -p 16V100 -o 'SAI_PARTITION=%P state=%a nodes=%D gres=%G' diff --git a/ci/sai/run_gpu_case_attempts.sh b/ci/sai/run_gpu_case_attempts.sh new file mode 100755 index 00000000000..e536648f40c --- /dev/null +++ b/ci/sai/run_gpu_case_attempts.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 7 ]]; then + echo "Usage: $0 SUITE_DIR CASE_DIR CASE_NAME ABACUS RANKS OMP_THREADS TASK_ROOT" >&2 + exit 2 +fi + +suite_dir=$(realpath -e "$1") +case_dir=$(realpath -e "$2") +case_name=$3 +abacus=$(realpath -e "$4") +ranks=$5 +omp_threads=$6 +task_root=$(realpath -e "$7") +retry_delay=10 + +[[ $case_name =~ ^[A-Za-z0-9_.-]+$ ]] +[[ $case_dir == "$suite_dir/$case_name" ]] +[[ -x $abacus ]] +[[ $ranks =~ ^[1-9][0-9]*$ ]] +[[ $omp_threads =~ ^[1-9][0-9]*$ ]] +: "${RESULT_ROOT:?}" +result_root=$(realpath -e "$RESULT_ROOT") +[[ $task_root == "$result_root/"* ]] +autotest=$(realpath -e "$suite_dir/../integrate/Autotest.sh") +[[ -f $autotest ]] + +combined_log="$task_root/case.log" +metadata="$task_root/pmix-retry.tsv" +: > "$combined_log" + +cleanup_case_outputs() { + rm -rf "$case_dir/OUT.autotest" + rm -f "$case_dir/log.txt" "$case_dir/result.out" +} + +is_pmix_startup_failure() { + local log=$1 + grep -aEq 'PMIX_ERR_(FILE_OPEN_FAILURE|OUT_OF_RESOURCE)' "$log" && + grep -aFq 'MPI_Init_thread' "$log" && + grep -aFq 'PMIx_Init failed' "$log" +} + +attempt=1 +retried=0 +retry_reason=none +final_pmix=0 +test_rc=2 +while [[ $attempt -le 2 ]]; do + cleanup_case_outputs + attempt_log="$task_root/case-attempt-${attempt}.log" + printf 'SAI_GPU_CASE_ATTEMPT attempt=%s max=2\n' "$attempt" \ + | tee -a "$combined_log" + + set +e + (cd "$suite_dir" && + timeout --signal=TERM --kill-after=30s 10m \ + bash "$autotest" -a "$abacus" -n "$ranks" \ + -o "$omp_threads" -f CASES.task.txt -r "^${case_name}$") \ + 2>&1 | tee "$attempt_log" | tee -a "$combined_log" + pipeline_status=("${PIPESTATUS[@]}") + set -e + [[ ${pipeline_status[1]} -eq 0 && ${pipeline_status[2]} -eq 0 ]] + test_rc=${pipeline_status[0]} + + final_pmix=0 + case $test_rc in + 0|124|137|143) ;; + *) + if is_pmix_startup_failure "$attempt_log"; then + final_pmix=1 + fi + ;; + esac + if [[ $attempt -eq 1 && $final_pmix -eq 1 ]]; then + retried=1 + retry_reason=pmix_startup + echo "SAI_PMIX_STARTUP_RETRY delay_seconds=$retry_delay" \ + | tee -a "$combined_log" + sleep "$retry_delay" + attempt=2 + continue + fi + break +done + +metadata_tmp=$(mktemp "$task_root/.pmix-retry.XXXXXX") +{ + printf 'attempts\t%s\n' "$attempt" + printf 'retried\t%s\n' "$retried" + printf 'retry_reason\t%s\n' "$retry_reason" + printf 'final_pmix\t%s\n' "$final_pmix" + printf 'final_rc\t%s\n' "$test_rc" +} > "$metadata_tmp" +mv -T "$metadata_tmp" "$metadata" +echo "SAI_GPU_CASE_ATTEMPTS attempts=$attempt retried=$retried final_pmix=$final_pmix rc=$test_rc" +exit "$test_rc" diff --git a/ci/sai/run_gpu_case_matrix.sh b/ci/sai/run_gpu_case_matrix.sh new file mode 100755 index 00000000000..c305dea47c1 --- /dev/null +++ b/ci/sai/run_gpu_case_matrix.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${CI_SOURCE:?}" +: "${CONTROL_ROOT:?}" +: "${INSTALL_ROOT:?}" +: "${RESULT_ROOT:?}" +: "${TOOLCHAIN_FILE:?}" +: "${MP_PROFILE:?}" + +bash "$CONTROL_ROOT/prepare_gpu_case_matrix.sh" + +matrix_root="$RESULT_ROOT/case-matrix" +manifest_root="$matrix_root/manifests" +log_root="$matrix_root/logs" +mkdir -p "$log_root" + +classes=(gpu1 gpu2 gpu4) +declare -A limits=([gpu1]=2 [gpu2]=8 [gpu4]=8) +declare -A ranks=([gpu1]=1 [gpu2]=2 [gpu4]=4) +declare -A qos=([gpu1]=flood-1o2gpu [gpu2]=flood-1o2gpu [gpu4]=flood-gpu) +declare -A job_ids=() +declare -A task_counts=() +declare -A job_names=() +declare -A job_id_files=() +active=1 +submit_pid= + +cancel_arrays() { + local class job_id_file recovered_id pid + local -a submit_pids=() + local -A seen=() + if [[ $active -eq 1 ]]; then + for pid in "$submit_pid" $(jobs -pr); do + if [[ -n $pid && -z ${seen[$pid]+x} ]]; then + submit_pids+=("$pid") + seen[$pid]=1 + fi + done + for pid in "${submit_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + fi + done + for pid in "${submit_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + submit_pid= + + for class in "${classes[@]}"; do + job_id_file=${job_id_files[$class]:-} + if [[ -z ${job_ids[$class]:-} && -n $job_id_file && -s $job_id_file ]]; then + recovered_id=$(<"$job_id_file") + recovered_id=${recovered_id%%;*} + if [[ $recovered_id =~ ^[0-9]+$ ]]; then + job_ids[$class]=$recovered_id + fi + fi + if [[ -n ${job_ids[$class]:-} ]]; then + scancel "${job_ids[$class]}" || true + elif [[ -n ${job_names[$class]:-} ]]; then + scancel --user="$USER" --name="${job_names[$class]}" || true + fi + done + fi +} +cleanup_job_id_files() { + local class + for class in "${classes[@]}"; do + if [[ -n ${job_id_files[$class]:-} ]]; then + rm -f "${job_id_files[$class]}" + fi + done +} +terminate() { + cancel_arrays + active=0 + cleanup_job_id_files + exit 143 +} +trap terminate HUP INT TERM +trap 'cancel_arrays; cleanup_job_id_files' EXIT + +jobs_file="$matrix_root/array-jobs.tsv" +: > "$jobs_file" +for class in "${classes[@]}"; do + manifest="$manifest_root/$class.tsv" + count=$(wc -l < "$manifest") + [[ $count -gt 0 ]] + task_counts[$class]=$count + last_task=$((count - 1)) + job_name="gha-${GITHUB_RUN_ID:-manual}-${class}-$$" + job_names[$class]=$job_name + job_id_file=$(mktemp "$matrix_root/.${class}-jobid.XXXXXX") + job_id_files[$class]=$job_id_file + export GPU_CASE_CLASS=$class + export GPU_CASE_RANKS=${ranks[$class]} + export GPU_CASE_MANIFEST=$manifest + sbatch --parsable \ + --job-name="$job_name" \ + --array="0-${last_task}%${limits[$class]}" \ + --partition=16V100 \ + --qos="${qos[$class]}" \ + --nodes=1 \ + --ntasks="${ranks[$class]}" \ + --gpus-per-node="${ranks[$class]}" \ + --time=00:15:00 \ + --chdir="$CI_SOURCE" \ + --output="$log_root/${class}-%A_%a.out" \ + --export=ALL \ + "$CONTROL_ROOT/test_gpu_case.sh" > "$job_id_file" & + submit_pid=$! + wait "$submit_pid" + submit_pid= + job_id=$(<"$job_id_file") + job_id=${job_id%%;*} + [[ $job_id =~ ^[0-9]+$ ]] + job_ids[$class]=$job_id + printf '%s\t%s\t%s\n' "$class" "$job_id" "$count" | tee -a "$jobs_file" +done + +job_list="${job_ids[gpu1]},${job_ids[gpu2]},${job_ids[gpu4]}" +queue_failures=0 +while true; do + if ! queue_output=$(squeue --noheader --jobs="$job_list"); then + queue_failures=$((queue_failures + 1)) + echo "squeue failed for array jobs (attempt $queue_failures/6)" >&2 + if [[ $queue_failures -ge 6 ]]; then + echo "Unable to prove that the GPU arrays left the queue" >&2 + exit 1 + fi + sleep 10 + continue + fi + queue_failures=0 + if [[ -z $queue_output ]]; then + break + fi + sleep 10 +done + +terminal_states='BOOT_FAIL|CANCELLED|COMPLETED|DEADLINE|FAILED|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|REVOKED|SPECIAL_EXIT|TIMEOUT' +accounting_ready=0 +accounting_output= +declare -A final_states=() +declare -A final_exit_codes=() +for accounting_attempt in {1..30}; do + if accounting_output=$(sacct --noheader --allocations --jobs="$job_list" \ + --parsable2 --format=JobID,State,ExitCode); then + final_states=() + final_exit_codes=() + while IFS='|' read -r got_job_id got_state got_exit_code; do + [[ -n ${got_job_id:-} ]] || continue + got_state=${got_state%% *} + got_state=${got_state%%+} + final_states[$got_job_id]=$got_state + final_exit_codes[$got_job_id]=$got_exit_code + done <<< "$accounting_output" + + accounting_ready=1 + for class in "${classes[@]}"; do + for ((task_id = 0; task_id < task_counts[$class]; task_id++)); do + task_job_id="${job_ids[$class]}_${task_id}" + task_state=${final_states[$task_job_id]:-} + if [[ ! $task_state =~ ^($terminal_states)$ ]]; then + accounting_ready=0 + break 2 + fi + done + done + if [[ $accounting_ready -eq 1 ]]; then + break + fi + else + echo "sacct failed while verifying array completion (attempt $accounting_attempt/30)" >&2 + fi + sleep 10 +done + +if [[ $accounting_ready -ne 1 ]]; then + echo "Unable to prove a terminal Slurm state for every GPU array task" >&2 + exit 1 +fi + +accounting_file="$matrix_root/array-task-accounting.tsv" +: > "$accounting_file" +for class in "${classes[@]}"; do + for ((task_id = 0; task_id < task_counts[$class]; task_id++)); do + task_job_id="${job_ids[$class]}_${task_id}" + printf '%s\t%s\t%s\t%s\t%s\n' \ + "$class" "$task_id" "${job_ids[$class]}" \ + "${final_states[$task_job_id]}" "${final_exit_codes[$task_job_id]}" \ + >> "$accounting_file" + done +done +printf 'SLURM_ARRAYS_TERMINAL=%s\n' "$job_list" \ + > "$matrix_root/arrays-terminal.txt" + +active=0 +trap - HUP INT TERM EXIT +cleanup_job_id_files + +if ! sacct --noheader --allocations --jobs="$job_list" \ + --format=JobID,JobIDRaw,JobName,State,ExitCode,Elapsed,NodeList \ + > "$matrix_root/array-sacct.txt"; then + printf '%s\n' "$accounting_output" > "$matrix_root/array-sacct.txt" +fi + +bash "$CONTROL_ROOT/summarize_gpu_case_matrix.sh" diff --git a/ci/sai/run_gpu_validation.sh b/ci/sai/run_gpu_validation.sh new file mode 100755 index 00000000000..2d84ef58d01 --- /dev/null +++ b/ci/sai/run_gpu_validation.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${CI_SOURCE:?}" +: "${CONTROL_ROOT:?}" +: "${INSTALL_ROOT:?}" +: "${RESULT_ROOT:?}" +: "${TOOLCHAIN_FILE:?}" +: "${MP_PROFILE:?}" + +unset SAI_SLURM_NODELIST + +matrix_log="$RESULT_ROOT/gpu-case-matrix-coordinator.log" +multinode_log="$RESULT_ROOT/cusolvermp-multinode-coordinator.log" +multinode_submit_log="$RESULT_ROOT/sai-cusolvermp-multinode-submit.log" +components_file="$RESULT_ROOT/gpu-validation-components.tsv" +matrix_pid= +multinode_pid= +children_active=1 + +cancel_children() { + local pid + local -a child_pids=() + local -A seen=() + if [[ $children_active -eq 1 ]]; then + for pid in "$matrix_pid" "$multinode_pid" $(jobs -pr); do + if [[ -n $pid && -z ${seen[$pid]+x} ]]; then + child_pids+=("$pid") + seen[$pid]=1 + fi + done + for pid in "${child_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + fi + done + for pid in "${child_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + fi +} +terminate() { + cancel_children + children_active=0 + exit 143 +} +trap terminate HUP INT TERM +trap cancel_children EXIT + +echo "Submitting GPU case arrays and 2-node cuSolverMp validation concurrently" +bash "$CONTROL_ROOT/run_gpu_case_matrix.sh" > "$matrix_log" 2>&1 & +matrix_pid=$! +"$CONTROL_ROOT/run_slurm_job.sh" "$multinode_submit_log" \ + "$RESULT_ROOT/sai-cusolvermp-multinode-%j.out" \ + "$CONTROL_ROOT/test_gpu.sbatch" > "$multinode_log" 2>&1 & +multinode_pid=$! +printf 'GPU_MATRIX_COORDINATOR_PID=%s\n' "$matrix_pid" +printf 'CUSOLVERMP_MULTINODE_COORDINATOR_PID=%s\n' "$multinode_pid" + +set +e +wait "$matrix_pid" +matrix_rc=$? +wait "$multinode_pid" +multinode_rc=$? +set -e +children_active=0 +trap - HUP INT TERM EXIT + +printf 'component\texit_code\ncase-matrix\t%s\ncusolvermp-multinode\t%s\n' \ + "$matrix_rc" "$multinode_rc" > "$components_file" + +echo "::group::SAI GPU case matrix coordinator" +cat "$matrix_log" +echo "::endgroup::" +echo "::group::SAI 2-node cuSolverMp coordinator" +cat "$multinode_log" +echo "::endgroup::" + +if [[ $matrix_rc -ne 0 ]]; then + echo "::error title=SAI GPU case matrix failed::Coordinator exit code $matrix_rc" +fi +if [[ $multinode_rc -ne 0 ]]; then + echo "::error title=SAI cuSolverMp multinode validation failed::Coordinator exit code $multinode_rc" +fi + +echo "SAI_GPU_VALIDATION_RESULT matrix_rc=$matrix_rc multinode_rc=$multinode_rc" +[[ $matrix_rc -eq 0 && $multinode_rc -eq 0 ]] diff --git a/ci/sai/run_remote_ci.sh b/ci/sai/run_remote_ci.sh new file mode 100755 index 00000000000..2e262e8057b --- /dev/null +++ b/ci/sai/run_remote_ci.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "Usage: $0 PROJECT_ROOT RUN_ROOT SOURCE_SHA CONTROL_SHA GITHUB_RUN_ID GITHUB_RUN_ATTEMPT" >&2 + exit 2 +fi + +export SAI_PROJECT_ROOT=$1 +export RUN_ROOT=$2 +source_sha=$3 +control_sha=$4 +export GITHUB_RUN_ID=$5 +export GITHUB_RUN_ATTEMPT=$6 +export CI_SOURCE=$RUN_ROOT/source +export CONTROL_ROOT=$RUN_ROOT/control +export BUILD_ROOT=$RUN_ROOT/build +export INSTALL_ROOT=$RUN_ROOT/install +export RESULT_ROOT=$RUN_ROOT/results +export TOOLCHAIN_FILE=$CONTROL_ROOT/toolchains/archive-mp09-sai-nccl2293.env.example +export MP_PROFILE=archive-mp09-sai-nccl2293 +export SAI_DISABLE_NCCL_IB=false +export SAI_NVIDIA_MP_ROOT=$SAI_PROJECT_ROOT/vendor/nvidia-mp-0.9-archive +export SAI_CUSOLVERMP_ROOT=$SAI_NVIDIA_MP_ROOT/libcusolvermp-linux-x86_64-0.9.0.6427_cuda12-archive +export SAI_CUBLASMP_ROOT=$SAI_NVIDIA_MP_ROOT/libcublasmp-linux-x86_64-0.9.1.3056_cuda12-archive +unset GITHUB_ENV GITHUB_STEP_SUMMARY SAI_SLURM_NODELIST + +[[ $SAI_PROJECT_ROOT == "$HOME/"* ]] +[[ $RUN_ROOT == "$SAI_PROJECT_ROOT/runs/"* ]] +run_parent=$(dirname "$RUN_ROOT") +[[ $(dirname "$run_parent") == "$SAI_PROJECT_ROOT/runs" ]] +run_namespace=${run_parent##*/} +[[ $run_namespace =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] +[[ -d $CI_SOURCE && -d $CONTROL_ROOT && -d $BUILD_ROOT && \ + -d $INSTALL_ROOT && -d $RESULT_ROOT ]] +[[ $source_sha =~ ^[0-9a-fA-F]{40}$ ]] +[[ $control_sha =~ ^[0-9a-fA-F]{40}$ ]] +for control_file in prepare_nvidia_mp.sh prepare_cusolvermp_smoke.sh \ + run_slurm_job.sh run_gpu_case_attempts.sh build_gpu.sbatch \ + run_gpu_validation.sh; do + [[ -f $CONTROL_ROOT/$control_file ]] +done + +child_pid= +child_active=0 +cancel_child() { + local pid + local -a child_pids=() + local -A seen=() + if [[ $child_active -eq 1 ]]; then + for pid in "$child_pid" $(jobs -pr); do + if [[ -n $pid && -z ${seen[$pid]+x} ]]; then + child_pids+=("$pid") + seen[$pid]=1 + fi + done + for pid in "${child_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + fi + done + for pid in "${child_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + fi +} +terminate() { + cancel_child + child_active=0 + exit 143 +} +trap terminate HUP INT TERM +trap cancel_child EXIT + +{ + printf 'source_sha\t%s\n' "$source_sha" + printf 'control_sha\t%s\n' "$control_sha" + printf 'github_run_id\t%s\n' "$GITHUB_RUN_ID" + printf 'github_run_attempt\t%s\n' "$GITHUB_RUN_ATTEMPT" + printf 'remote_user\t%s\n' "$USER" + printf 'remote_host\t%s\n' "$(hostname)" + printf 'project_root\t%s\n' "$SAI_PROJECT_ROOT" + printf 'run_namespace\t%s\n' "$run_namespace" +} > "$RESULT_ROOT/remote-run-metadata.tsv" + +bash "$CONTROL_ROOT/prepare_nvidia_mp.sh" + +child_active=1 +bash "$CONTROL_ROOT/run_slurm_job.sh" \ + "$RESULT_ROOT/sai-build-submit.log" \ + "$RESULT_ROOT/sai-build-%j.out" \ + "$CONTROL_ROOT/build_gpu.sbatch" & +child_pid=$! +wait "$child_pid" +child_active=0 +child_pid= + +child_active=1 +bash "$CONTROL_ROOT/run_gpu_validation.sh" & +child_pid=$! +set +e +wait "$child_pid" +validation_rc=$? +set -e +child_active=0 +child_pid= + +trap - HUP INT TERM EXIT +echo "SAI_REMOTE_CI_RESULT validation_rc=$validation_rc source_sha=$source_sha" +exit "$validation_rc" diff --git a/ci/sai/run_slurm_job.sh b/ci/sai/run_slurm_job.sh new file mode 100755 index 00000000000..671c5777103 --- /dev/null +++ b/ci/sai/run_slurm_job.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 SUBMIT_LOG OUTPUT_PATTERN JOB_SCRIPT" >&2 + exit 2 +fi + +submit_log=$1 +output_pattern=$2 +job_script=$3 +job_id= +job_active=0 +submit_pid= +job_name="gha-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-$$" +sbatch_args=() +nodelist_pattern='^[[:alnum:],._-]+$' + +if [[ -n ${SAI_SLURM_NODELIST:-} ]]; then + if [[ ! $SAI_SLURM_NODELIST =~ $nodelist_pattern ]]; then + printf 'Invalid SAI_SLURM_NODELIST: %q\n' "$SAI_SLURM_NODELIST" >&2 + exit 2 + fi + sbatch_args+=(--nodelist="$SAI_SLURM_NODELIST") +fi + +submit_dir=$(realpath -e "$(dirname "$submit_log")") +[[ $submit_dir == "$HOME/"* ]] +job_id_file=$(mktemp "$submit_dir/.slurm-jobid.XXXXXX") + +cancel_job() { + local pid + local -a submit_pids=() + local -A seen=() + for pid in "$submit_pid" $(jobs -pr); do + if [[ -n $pid && -z ${seen[$pid]+x} ]]; then + submit_pids+=("$pid") + seen[$pid]=1 + fi + done + for pid in "${submit_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + fi + done + for pid in "${submit_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + submit_pid= + + if [[ -z $job_id && -s $job_id_file ]]; then + job_id=$(<"$job_id_file") + job_id=${job_id%%;*} + [[ $job_id =~ ^[0-9]+$ ]] || job_id= + fi + if [[ $job_active -eq 1 && -n $job_id ]]; then + echo "Cancelling Slurm job $job_id" | tee -a "$submit_log" + scancel "$job_id" || true + elif [[ $job_active -eq 1 ]]; then + echo "Cancelling Slurm job name $job_name" | tee -a "$submit_log" + scancel --user="$USER" --name="$job_name" || true + fi + rm -f "$job_id_file" +} +terminate() { + cancel_job + job_active=0 + exit 143 +} +trap terminate HUP INT TERM +trap cancel_job EXIT + +job_active=1 +sbatch --parsable --export=ALL --job-name="$job_name" \ + --chdir="${CI_SOURCE:?}" \ + --output="$output_pattern" "${sbatch_args[@]}" \ + "$job_script" > "$job_id_file" & +submit_pid=$! +wait "$submit_pid" +submit_pid= +job_id=$(<"$job_id_file") +rm -f "$job_id_file" +job_id=${job_id%%;*} +[[ $job_id =~ ^[0-9]+$ ]] +printf 'SLURM_JOB_ID=%s\n' "$job_id" | tee "$submit_log" + +record= +while true; do + record=$(sacct --noheader --allocations --jobs="$job_id" \ + --format=JobIDRaw,State,ExitCode \ + | awk -v id="$job_id" '$1 == id {record=$2 " " $3} END {print record}') + state=${record%% *} + state=${state%%+} + exit_code=${record#* } + case $state in + BOOT_FAIL|CANCELLED|COMPLETED|DEADLINE|FAILED|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|REVOKED|SPECIAL_EXIT|TIMEOUT) + break + ;; + esac + sleep 10 +done +job_active=0 +trap - HUP INT TERM EXIT +rm -f "$job_id_file" +printf 'SLURM_FINAL=%s\n' "$record" | tee -a "$submit_log" +[[ $state == COMPLETED && $exit_code == 0:0 ]] diff --git a/ci/sai/source_transfer_cache.sh b/ci/sai/source_transfer_cache.sh new file mode 100755 index 00000000000..96979bc5dc8 --- /dev/null +++ b/ci/sai/source_transfer_cache.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROJECT_ROOT= +RUN_ROOT= +RUN_NAME= +RUN_SOURCE= +CACHE_ROOT= +SNAPSHOT_ROOT= +TRANSFER_PARENT= +TRANSFER_ROOT= +TRANSFER_SOURCE= +TRANSFER_MARKER= +CACHE_ROLE= + +resolve_run() { + local canonical_home requested_project=$1 requested_run=$2 run_parent + canonical_home=$(cd "$HOME" && pwd -P) + PROJECT_ROOT=$(realpath -e "$requested_project") + RUN_ROOT=$(realpath -e "$requested_run") + [[ $PROJECT_ROOT == "$canonical_home/"* ]] + [[ $RUN_ROOT == "$PROJECT_ROOT/runs/"* ]] + run_parent=$(dirname "$RUN_ROOT") + if [[ $run_parent != "$PROJECT_ROOT/runs" ]]; then + [[ $(dirname "$run_parent") == "$PROJECT_ROOT/runs" ]] + [[ ${run_parent##*/} =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] + fi + RUN_NAME=${RUN_ROOT##*/} + [[ $RUN_NAME =~ ^[0-9]+-[0-9]+$ ]] + [[ -f $RUN_ROOT/.ci-created && ! -L $RUN_ROOT/.ci-created ]] + RUN_SOURCE=$(realpath -e "$RUN_ROOT/source") + [[ $RUN_SOURCE == "$RUN_ROOT/source" ]] +} + +resolve_cache() { + CACHE_ROOT=$(realpath -e "$PROJECT_ROOT/cache") + SNAPSHOT_ROOT=$(realpath -e "$CACHE_ROOT/source-snapshots") + TRANSFER_PARENT=$(realpath -e "$CACHE_ROOT/source-transfers") + [[ $CACHE_ROOT == "$PROJECT_ROOT/cache" ]] + [[ $SNAPSHOT_ROOT == "$CACHE_ROOT/source-snapshots" ]] + [[ $TRANSFER_PARENT == "$CACHE_ROOT/source-transfers" ]] +} + +resolve_transfer() { + local requested_transfer=$1 + resolve_cache + TRANSFER_ROOT=$(realpath -e "$requested_transfer") + [[ $TRANSFER_ROOT == "$TRANSFER_PARENT/$RUN_NAME" ]] + [[ -d $TRANSFER_ROOT && ! -L $TRANSFER_ROOT ]] + TRANSFER_SOURCE=$(realpath -e "$TRANSFER_ROOT/source") + [[ $TRANSFER_SOURCE == "$TRANSFER_ROOT/source" ]] + [[ -d $TRANSFER_SOURCE && ! -L $TRANSFER_SOURCE ]] + TRANSFER_MARKER="$TRANSFER_ROOT/.ci-source-transfer" + [[ -f $TRANSFER_MARKER && ! -L $TRANSFER_MARKER ]] +} + +verify_marker() { + local source_sha=$1 role + grep -Fxq "run_root=$RUN_ROOT" "$TRANSFER_MARKER" + grep -Fxq "source_sha=$source_sha" "$TRANSFER_MARKER" + role=$(awk -F= '$1 == "cache_role" {print $2}' "$TRANSFER_MARKER") + [[ $role == baseline || $role == candidate ]] + CACHE_ROLE=$role +} + +lock_cache() { + exec 8<"$CACHE_ROOT" + flock 8 +} + +verify_tree() { + local tree_root=$1 manifest=$2 record metadata path mode type object + local parent relative actual permission hash_output offset batch_count index + declare -A expected_mode=() + declare -A expected_object=() + declare -A expected_dir=() + declare -A actual_permission=() + local -a regular_paths=() + local -a regular_files=() + local -a batch_hashes=() + + [[ -d $tree_root && ! -L $tree_root ]] || return 1 + [[ -f $manifest && ! -L $manifest ]] || return 1 + gzip -t -- "$manifest" || return 1 + + while IFS= read -r -d '' record; do + [[ $record == *$'\t'* ]] || return 1 + metadata=${record%%$'\t'*} + path=${record#*$'\t'} + [[ $metadata =~ ^([0-9]{6})\ ([a-z]+)\ ([0-9a-f]{40})$ ]] || return 1 + mode=${BASH_REMATCH[1]} + type=${BASH_REMATCH[2]} + object=${BASH_REMATCH[3]} + [[ $type == blob ]] || return 1 + [[ $mode == 100644 || $mode == 100755 || $mode == 120000 ]] || return 1 + [[ -n $path && $path != /* ]] || return 1 + case "/$path/" in + */./*|*/../*|*//*) return 1 ;; + esac + [[ -z ${expected_mode["$path"]+present} ]] || return 1 + [[ -z ${expected_dir["$path"]+present} ]] || return 1 + expected_mode["$path"]=$mode + expected_object["$path"]=$object + + parent=$path + while [[ $parent == */* ]]; do + parent=${parent%/*} + [[ -z ${expected_mode["$parent"]+present} ]] || return 1 + expected_dir["$parent"]=1 + done + record= + done < <(gzip -cd -- "$manifest") + [[ -z $record ]] || return 1 + + while IFS= read -r -d '' permission && IFS= read -r -d '' actual; do + relative=${actual#"$tree_root"/} + if [[ -d $actual && ! -L $actual ]]; then + [[ -n ${expected_dir["$relative"]+present} ]] || return 1 + elif [[ -f $actual && ! -L $actual ]] || [[ -L $actual ]]; then + [[ -n ${expected_mode["$relative"]+present} ]] || return 1 + actual_permission["$relative"]=$permission + else + echo "Unsupported source-cache filesystem entry: $relative" >&2 + return 1 + fi + done < <(find "$tree_root" -mindepth 1 -printf '%m\0%p\0') + + for path in "${!expected_mode[@]}"; do + actual="$tree_root/$path" + mode=${expected_mode["$path"]} + if [[ $mode == 120000 ]]; then + [[ -L $actual ]] || return 1 + object=$(readlink -n -- "$actual" | git hash-object --stdin) || return 1 + [[ $object == "${expected_object["$path"]}" ]] || return 1 + else + [[ -f $actual && ! -L $actual ]] || return 1 + permission=${actual_permission["$path"]} + if [[ $mode == 100755 ]]; then + (( (8#$permission & 8#111) != 0 )) || return 1 + else + (( (8#$permission & 8#111) == 0 )) || return 1 + fi + regular_paths+=("$path") + regular_files+=("$actual") + fi + done + + for ((offset = 0; offset < ${#regular_files[@]}; offset += 256)); do + batch_count=$((${#regular_files[@]} - offset)) + (( batch_count > 256 )) && batch_count=256 + hash_output=$(git hash-object --no-filters -- \ + "${regular_files[@]:offset:batch_count}") || return 1 + mapfile -t batch_hashes <<< "$hash_output" + [[ ${#batch_hashes[@]} -eq $batch_count ]] || return 1 + for ((index = 0; index < batch_count; index++)); do + path=${regular_paths[offset + index]} + [[ ${batch_hashes[index]} == "${expected_object["$path"]}" ]] || return 1 + done + done +} + +quarantine_latest() { + local latest=$1 reason=$2 quarantine + quarantine="$CACHE_ROOT/.source-latest.invalid.$RUN_NAME" + [[ ! -e $quarantine && ! -L $quarantine ]] + mv -T -- "$latest" "$quarantine" + printf 'SOURCE_CACHE_INVALID reason=%s quarantined=%s\n' \ + "$reason" "$quarantine" >&2 +} + +cleanup_orphan_snapshots() { + local current_name=$1 candidate name manifest + while IFS= read -r -d '' candidate; do + name=${candidate##*/} + [[ $name =~ ^[0-9a-f]{40}\.[0-9]+-[0-9]+$ ]] || continue + [[ $name == "$current_name" ]] && continue + candidate=$(realpath -e "$candidate") + [[ $candidate == "$SNAPSHOT_ROOT/$name" ]] + rm -rf --one-file-system -- "$candidate" + manifest="$SNAPSHOT_ROOT/$name.manifest.gz" + if [[ -f $manifest && ! -L $manifest ]]; then + rm -f -- "$manifest" + fi + done < <(find "$SNAPSHOT_ROOT" -mindepth 1 -maxdepth 1 -type d -print0) + + while IFS= read -r -d '' manifest; do + name=${manifest##*/} + [[ $name =~ ^([0-9a-f]{40}\.[0-9]+-[0-9]+)\.manifest\.gz$ ]] || continue + [[ ${BASH_REMATCH[1]} == "$current_name" ]] && continue + [[ -f $manifest && ! -L $manifest ]] + rm -f -- "$manifest" + done < <(find "$SNAPSHOT_ROOT" -mindepth 1 -maxdepth 1 \ + -type f -name '*.manifest.gz' -print0) +} + +prepare_transfer() { + local requested_project=$1 requested_run=$2 source_sha=$3 + local cache_role=$4 + local base_sha=none latest latest_name='' base_snapshot manifest cache_missing + local latest_valid=0 cache_invalid=0 + [[ $source_sha =~ ^[0-9a-f]{40}$ ]] + [[ $cache_role == baseline || $cache_role == candidate ]] + resolve_run "$requested_project" "$requested_run" + [[ -z $(find "$RUN_SOURCE" -mindepth 1 -maxdepth 1 -print -quit) ]] + + cache_missing=$(realpath --canonicalize-missing "$PROJECT_ROOT/cache") + [[ $cache_missing == "$PROJECT_ROOT/cache" ]] + mkdir -p "$cache_missing/source-snapshots" "$cache_missing/source-transfers" + resolve_cache + lock_cache + + TRANSFER_ROOT="$TRANSFER_PARENT/$RUN_NAME" + [[ ! -e $TRANSFER_ROOT && ! -L $TRANSFER_ROOT ]] || { + echo "Refusing to reuse source transfer: $TRANSFER_ROOT" >&2 + exit 1 + } + mkdir -p "$TRANSFER_ROOT/source" + { + printf 'run_root=%s\n' "$RUN_ROOT" + printf 'source_sha=%s\n' "$source_sha" + printf 'cache_role=%s\n' "$cache_role" + } > "$TRANSFER_ROOT/.ci-source-transfer" + + latest="$CACHE_ROOT/source-latest" + if [[ -e $latest || -L $latest ]]; then + if [[ -f $latest && ! -L $latest ]]; then + IFS= read -r latest_name < "$latest" || true + if [[ $latest_name =~ ^([0-9a-f]{40})\.([0-9]+-[0-9]+)$ ]] && + printf '%s\n' "$latest_name" | cmp -s - "$latest"; then + base_sha=${BASH_REMATCH[1]} + base_snapshot="$SNAPSHOT_ROOT/$latest_name" + manifest="$SNAPSHOT_ROOT/$latest_name.manifest.gz" + if [[ -d $base_snapshot && ! -L $base_snapshot && + -f $manifest && ! -L $manifest ]] && + verify_tree "$base_snapshot" "$manifest"; then + base_snapshot=$(realpath -e "$base_snapshot") + [[ $base_snapshot == "$SNAPSHOT_ROOT/$latest_name" ]] + latest_valid=1 + else + base_sha=none + cache_invalid=1 + if [[ $cache_role == baseline ]]; then + quarantine_latest "$latest" content_or_manifest_mismatch + else + echo "SOURCE_CACHE_INVALID reason=content_or_manifest_mismatch role=candidate" >&2 + fi + fi + else + cache_invalid=1 + if [[ $cache_role == baseline ]]; then + quarantine_latest "$latest" malformed_pointer + else + echo "SOURCE_CACHE_INVALID reason=malformed_pointer role=candidate" >&2 + fi + fi + else + cache_invalid=1 + if [[ $cache_role == baseline ]]; then + quarantine_latest "$latest" unsafe_pointer_type + else + echo "SOURCE_CACHE_INVALID reason=unsafe_pointer_type role=candidate" >&2 + fi + fi + fi + + if [[ $latest_valid -eq 1 ]]; then + if [[ $cache_role == baseline ]]; then + cleanup_orphan_snapshots "$latest_name" + fi + cp -a "$base_snapshot/." "$TRANSFER_ROOT/source/" + elif [[ $cache_invalid -eq 0 && $cache_role == baseline ]]; then + cleanup_orphan_snapshots "" + fi + printf 'SOURCE_TRANSFER_ROOT=%s\n' "$TRANSFER_ROOT" + printf 'SOURCE_CACHE_BASE_SHA=%s\n' "$base_sha" + printf 'SOURCE_CACHE_ROLE=%s\n' "$cache_role" +} + +receive_payload() { + local requested_project=$1 requested_run=$2 requested_transfer=$3 + local mode=$4 source_sha=$5 payload manifest patch_file + [[ $source_sha =~ ^[0-9a-f]{40}$ ]] + [[ $mode == full || $mode == delta ]] + resolve_run "$requested_project" "$requested_run" + resolve_transfer "$requested_transfer" + verify_marker "$source_sha" + payload="$TRANSFER_ROOT/source-payload.gz" + manifest="$TRANSFER_ROOT/source-manifest.gz" + [[ -f $payload && ! -L $payload ]] + [[ -f $manifest && ! -L $manifest ]] + gzip -t -- "$manifest" + + if [[ $mode == full ]]; then + find "$TRANSFER_SOURCE" -mindepth 1 -maxdepth 1 \ + -exec rm -rf --one-file-system -- {} + + fi + patch_file="$TRANSFER_ROOT/source.patch" + gzip -cd "$payload" > "$patch_file" + if [[ -s $patch_file ]]; then + (cd "$TRANSFER_SOURCE" && git apply --check --binary \ + --whitespace=nowarn "$patch_file") + (cd "$TRANSFER_SOURCE" && git apply --binary \ + --whitespace=nowarn "$patch_file") + fi + rm -f "$patch_file" + rm -f "$payload" + printf 'SOURCE_PAYLOAD_APPLIED mode=%s sha=%s\n' "$mode" "$source_sha" +} + +finalize_transfer() { + local requested_project=$1 requested_run=$2 requested_transfer=$3 + local source_sha=$4 snapshot_name snapshot manifest snapshot_manifest + local latest_tmp + [[ $source_sha =~ ^[0-9a-f]{40}$ ]] + resolve_run "$requested_project" "$requested_run" + resolve_transfer "$requested_transfer" + verify_marker "$source_sha" + lock_cache + [[ ! -e $TRANSFER_ROOT/source-payload.gz ]] + [[ -z $(find "$RUN_SOURCE" -mindepth 1 -maxdepth 1 -print -quit) ]] + manifest="$TRANSFER_ROOT/source-manifest.gz" + verify_tree "$TRANSFER_SOURCE" "$manifest" + cp -a "$TRANSFER_SOURCE/." "$RUN_SOURCE/" + + if [[ $CACHE_ROLE == candidate ]]; then + rm -rf --one-file-system -- "$TRANSFER_SOURCE" + rm -f "$manifest" "$TRANSFER_MARKER" + rmdir "$TRANSFER_ROOT" + printf 'SOURCE_CACHE_PROMOTION=skipped role=candidate source_sha=%s\n' \ + "$source_sha" + return + fi + + snapshot_name="$source_sha.$RUN_NAME" + snapshot="$SNAPSHOT_ROOT/$snapshot_name" + snapshot_manifest="$SNAPSHOT_ROOT/$snapshot_name.manifest.gz" + [[ ! -e $snapshot && ! -L $snapshot ]] + [[ ! -e $snapshot_manifest && ! -L $snapshot_manifest ]] + mv -T "$TRANSFER_SOURCE" "$snapshot" + mv -T "$manifest" "$snapshot_manifest" + latest_tmp=$(mktemp "$CACHE_ROOT/.source-latest.XXXXXX") + printf '%s\n' "$snapshot_name" > "$latest_tmp" + mv -T "$latest_tmp" "$CACHE_ROOT/source-latest" + rm -f "$TRANSFER_MARKER" + rmdir "$TRANSFER_ROOT" + + cleanup_orphan_snapshots "$snapshot_name" + printf 'SOURCE_CACHE_PROMOTED_SHA=%s\n' "$source_sha" + printf 'SOURCE_CACHE_SNAPSHOT=%s\n' "$snapshot" +} + +command=${1:-} +case $command in + prepare) + [[ $# -eq 5 ]] || { echo "Usage: $0 prepare PROJECT_ROOT RUN_ROOT SOURCE_SHA baseline|candidate" >&2; exit 2; } + prepare_transfer "$2" "$3" "$4" "$5" + ;; + receive) + [[ $# -eq 6 ]] || { echo "Usage: $0 receive PROJECT_ROOT RUN_ROOT TRANSFER_ROOT MODE SOURCE_SHA" >&2; exit 2; } + receive_payload "$2" "$3" "$4" "$5" "$6" + ;; + finalize) + [[ $# -eq 5 ]] || { echo "Usage: $0 finalize PROJECT_ROOT RUN_ROOT TRANSFER_ROOT SOURCE_SHA" >&2; exit 2; } + finalize_transfer "$2" "$3" "$4" "$5" + ;; + *) + echo "Usage: $0 {prepare|receive|finalize} ..." >&2 + exit 2 + ;; +esac diff --git a/ci/sai/summarize_gpu_case_matrix.sh b/ci/sai/summarize_gpu_case_matrix.sh new file mode 100755 index 00000000000..c310afe635d --- /dev/null +++ b/ci/sai/summarize_gpu_case_matrix.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${RESULT_ROOT:?}" + +matrix_root="$RESULT_ROOT/case-matrix" +manifest_root="$matrix_root/manifests" +jobs_file="$matrix_root/array-jobs.tsv" +accounting_file="$matrix_root/array-task-accounting.tsv" +[[ -f "$jobs_file" ]] +[[ -f "$accounting_file" ]] + +classes=(gpu1 gpu2 gpu4) +declare -A job_ids=() +declare -A task_counts=() +while IFS=$'\t' read -r class job_id count; do + [[ $class =~ ^gpu(1|2|4)$ ]] + [[ $job_id =~ ^[0-9]+$ ]] + [[ $count =~ ^[0-9]+$ ]] + job_ids[$class]=$job_id + task_counts[$class]=$count +done < "$jobs_file" + +declare -A slurm_states=() +declare -A slurm_exit_codes=() +accounting_rows=0 +while IFS=$'\t' read -r class task_id job_id slurm_state slurm_exit_code extra; do + [[ -z ${extra:-} ]] + [[ $class =~ ^gpu(1|2|4)$ ]] + [[ $task_id =~ ^[0-9]+$ ]] + [[ $job_id =~ ^[0-9]+$ ]] + [[ $job_id == "${job_ids[$class]:-}" ]] + [[ $slurm_state =~ ^(BOOT_FAIL|CANCELLED|COMPLETED|DEADLINE|FAILED|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|REVOKED|SPECIAL_EXIT|TIMEOUT)$ ]] + [[ $slurm_exit_code =~ ^[0-9]+:[0-9]+$ ]] + accounting_key="$class-$task_id" + [[ -z ${slurm_states[$accounting_key]+x} ]] + slurm_states[$accounting_key]=$slurm_state + slurm_exit_codes[$accounting_key]=$slurm_exit_code + accounting_rows=$((accounting_rows + 1)) +done < "$accounting_file" +[[ $accounting_rows -eq 48 ]] + +summary="$matrix_root/gpu-case-summary.md" +{ + echo "## SAI GPU case matrix" + echo + echo '| Resource | Test case | Status | Slurm task | Ranks | Elapsed |' + echo '| --- | --- | --- | ---: | ---: | ---: |' +} > "$summary" + +passed=0 +failed=0 +infra=0 +for class in "${classes[@]}"; do + [[ -n ${job_ids[$class]:-} ]] + manifest="$manifest_root/$class.tsv" + task_id=0 + while IFS=$'\t' read -r suite case_name; do + status_file="$matrix_root/status/${class}-${task_id}.tsv" + expected_ranks=${class#gpu} + expected_job_id=${job_ids[$class]} + accounting_key="$class-$task_id" + slurm_state=${slurm_states[$accounting_key]:-} + slurm_exit_code=${slurm_exit_codes[$accounting_key]:-} + status_error= + status_lines=() + got_class= + got_task= + got_suite= + got_case= + ranks= + state= + rc= + elapsed= + job_id= + + if [[ ! -f $status_file ]]; then + status_error=missing_status + elif [[ ! -s $status_file ]]; then + status_error=empty_status + else + mapfile -t status_lines < "$status_file" + if [[ ${#status_lines[@]} -ne 1 ]]; then + status_error=expected_one_status_row + elif [[ -n $(tail -c 1 "$status_file") ]]; then + status_error=missing_final_newline + elif [[ $(awk -F '\t' 'NR == 1 { print NF }' "$status_file") -ne 9 ]]; then + status_error=expected_nine_fields + else + IFS=$'\t' read -r got_class got_task got_suite got_case ranks state rc elapsed job_id \ + <<< "${status_lines[0]}" + errors=() + [[ $got_class == "$class" ]] || errors+=(class_mismatch) + [[ $got_task =~ ^[0-9]+$ ]] || errors+=(task_not_numeric) + [[ $got_task == "$task_id" ]] || errors+=(task_mismatch) + [[ $got_suite == "$suite" ]] || errors+=(suite_mismatch) + [[ $got_case == "$case_name" ]] || errors+=(case_mismatch) + [[ $ranks =~ ^[0-9]+$ ]] || errors+=(ranks_not_numeric) + [[ $ranks == "$expected_ranks" ]] || errors+=(ranks_mismatch) + [[ $state =~ ^(PASS|FAIL|TIMEOUT|INFRA)$ ]] || errors+=(invalid_state) + [[ $rc =~ ^[0-9]+$ ]] || errors+=(rc_not_numeric) + if [[ $state == PASS && $rc != 0 ]]; then + errors+=(pass_with_nonzero_rc) + elif [[ $state =~ ^(FAIL|TIMEOUT|INFRA)$ && $rc == 0 ]]; then + errors+=(failure_with_zero_rc) + fi + slurm_rc=${slurm_exit_code%%:*} + slurm_signal=${slurm_exit_code#*:} + if [[ $state == PASS ]]; then + [[ $slurm_state == COMPLETED ]] || errors+=(pass_without_completed_slurm_state) + [[ $slurm_exit_code == 0:0 ]] || errors+=(pass_with_nonzero_slurm_exit) + elif [[ $state =~ ^(FAIL|TIMEOUT|INFRA)$ ]]; then + [[ $slurm_state == FAILED ]] || errors+=(failure_without_failed_slurm_state) + [[ $slurm_rc == "$rc" && $slurm_signal == 0 ]] || errors+=(status_slurm_exit_mismatch) + fi + [[ $elapsed =~ ^[0-9]+$ ]] || errors+=(elapsed_not_numeric) + [[ $job_id =~ ^[0-9]+$ ]] || errors+=(job_id_not_numeric) + [[ $job_id == "$expected_job_id" ]] || errors+=(job_id_mismatch) + if [[ ${#errors[@]} -gt 0 ]]; then + status_error=$(IFS=,; echo "${errors[*]}") + fi + fi + fi + + if [[ -n $status_error ]]; then + state=INFRA + rc=$status_error + elapsed='-' + job_id=$expected_job_id + ranks=$expected_ranks + fi + + case $state in + PASS) passed=$((passed + 1)) ;; + FAIL|TIMEOUT) + failed=$((failed + 1)) + echo "::error title=SAI GPU case failed::${suite}/${case_name} state=${state} rc=${rc} Slurm=${job_id}_${task_id}" + ;; + INFRA) + infra=$((infra + 1)) + echo "::error title=SAI GPU case infrastructure failure::${suite}/${case_name} state=${state} rc=${rc} Slurm=${job_id}_${task_id}" + ;; + esac + printf '| %s | `%s/%s` | %s | `%s_%s` | %s | %s s |\n' \ + "$class" "$suite" "$case_name" "$state" "$job_id" "$task_id" \ + "$ranks" "$elapsed" >> "$summary" + task_id=$((task_id + 1)) + done < "$manifest" + [[ $task_id -eq ${task_counts[$class]} ]] +done + +{ + echo + echo "Passed: **$passed**; Failed: **$failed**; Infrastructure: **$infra**" +} >> "$summary" + +cat "$summary" +if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then + cat "$summary" >> "$GITHUB_STEP_SUMMARY" +fi + +echo "SAI_GPU_CASE_MATRIX_RESULT passed=$passed failed=$failed infra=$infra" +[[ $passed -eq 48 && $failed -eq 0 && $infra -eq 0 ]] diff --git a/ci/sai/test_gpu.sbatch b/ci/sai/test_gpu.sbatch new file mode 100755 index 00000000000..4606bc6f62e --- /dev/null +++ b/ci/sai/test_gpu.sbatch @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +#SBATCH --job-name=abacus-cusolvermp-multinode-ci +#SBATCH --partition=16V100 +#SBATCH --qos=flood-gpu +#SBATCH --nodes=2 +#SBATCH --ntasks=16 +#SBATCH --ntasks-per-node=8 +#SBATCH --gpus-per-node=8 +#SBATCH --time=00:40:00 + +set -euo pipefail + +: "${CI_SOURCE:?}" +: "${CONTROL_ROOT:?}" +: "${INSTALL_ROOT:?}" +: "${RESULT_ROOT:?}" +: "${TOOLCHAIN_FILE:?}" +: "${MP_PROFILE:?}" + +source "$TOOLCHAIN_FILE" +declare -F sai_load_toolchain >/dev/null || { + echo "Missing sai_load_toolchain() in $TOOLCHAIN_FILE" >&2 + exit 1 +} +sai_load_toolchain + +source "/opt/sai_config/mps_mapping.d/${SLURM_JOB_PARTITION}.bash" +case ${SAI_DISABLE_NCCL_IB-false} in + true) + export NCCL_IB_DISABLE=1 + ;; + false) + unset NCCL_IB_DISABLE + ;; + *) + echo "Invalid SAI_DISABLE_NCCL_IB=${SAI_DISABLE_NCCL_IB}" >&2 + exit 2 + ;; +esac +SAI_SYSTEM_MPIRUN=$(command -v mpirun) +export SAI_SYSTEM_MPIRUN MAP_OPT +launcher_dir="$RESULT_ROOT/launcher" +mkdir -p "$launcher_dir" +ln -sfn "$CONTROL_ROOT/mpirun_with_mapping.sh" "$launcher_dir/mpirun" +export PATH="$launcher_dir:$PATH" +export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" + +echo "SAI runtime mapping: partition=$SLURM_JOB_PARTITION tasks=$SLURM_NTASKS map_by=$MAP_OPT omp_threads=$OMP_NUM_THREADS" +scontrol show hostnames "$SLURM_JOB_NODELIST" | sed 's/^/SLURM_NODE=/' +env | grep -E '^(UCX|OMPI_MCA|PRTE_MCA|NCCL)_' | sort || true +if [[ -d /sys/class/infiniband ]]; then + find /sys/class/infiniband -mindepth 1 -maxdepth 1 -printf 'RDMA_DEVICE=%f\n' | sort +fi + +ABACUS="$INSTALL_ROOT/bin/abacus" +[[ -x "$ABACUS" ]] +ABACUS_LDD=$(ldd "$ABACUS") +printf '%s\n' "$ABACUS_LDD" +if grep -q 'not found' <<< "$ABACUS_LDD"; then + echo "ABACUS has unresolved runtime dependencies" >&2 + exit 1 +fi + +if [[ ${SAI_REQUIRE_NCCL_SAI_RAIL_BY_CHANNEL:-0} == 1 ]]; then + [[ ${NCCL_SAI_RAIL_BY_CHANNEL:-} == 1 ]] || { + echo "NCCL_SAI_RAIL_BY_CHANNEL=1 is required for this profile" >&2 + exit 1 + } + [[ -z ${LD_PRELOAD:-} ]] || { + printf 'LD_PRELOAD must be empty at runtime: %q\n' "$LD_PRELOAD" >&2 + exit 1 + } + NCCL_LOADED_PATH=$(awk '$1 == "libnccl.so.2" {print $3; exit}' <<< "$ABACUS_LDD") + [[ -n $NCCL_LOADED_PATH ]] + NCCL_EXPECTED_PATH=$(readlink -f "$SAI_NCCL_ROOT/lib/libnccl.so.2") + NCCL_LOADED_PATH=$(readlink -f "$NCCL_LOADED_PATH") + [[ $NCCL_LOADED_PATH == "$NCCL_EXPECTED_PATH" ]] || { + echo "Unexpected NCCL runtime: $NCCL_LOADED_PATH (expected $NCCL_EXPECTED_PATH)" >&2 + exit 1 + } + echo "SAI_NCCL_RUNTIME_VERIFIED=$NCCL_LOADED_PATH rail_by_channel=$NCCL_SAI_RAIL_BY_CHANNEL" +fi + +"$ABACUS" --info + +bash "$CONTROL_ROOT/prepare_cusolvermp_smoke.sh" +case_name=19_NO_Si48_CUSOLVERMP_TDDFT_GPU +case_dir="$RESULT_ROOT/cusolvermp-smoke/15_rtTDDFT_GPU/$case_name" +run_log="$RESULT_ROOT/cusolvermp-si48-run.log" +cd "$case_dir" +set +e +timeout --signal=TERM --kill-after=30s 35m \ + mpirun -np "$SLURM_NTASKS" "$ABACUS" 2>&1 | tee "$run_log" +suite_status=${PIPESTATUS[0]} +set -e +if [[ $suite_status -ne 0 ]]; then + echo "[ERROR] 2-node 16-GPU Si48 cuSolverMp RT-TDDFT case failed (rc=$suite_status)" +fi + +if [[ "$suite_status" -eq 0 ]]; then + echo "SAI_GPU_VALIDATION_PASSED profile=$MP_PROFILE" +else + echo "SAI_GPU_VALIDATION_FAILED profile=$MP_PROFILE" +fi +exit "$suite_status" diff --git a/ci/sai/test_gpu_case.sh b/ci/sai/test_gpu_case.sh new file mode 100755 index 00000000000..91c8d9a08d4 --- /dev/null +++ b/ci/sai/test_gpu_case.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${CI_SOURCE:?}" +: "${CONTROL_ROOT:?}" +: "${INSTALL_ROOT:?}" +: "${RESULT_ROOT:?}" +: "${TOOLCHAIN_FILE:?}" +: "${MP_PROFILE:?}" +: "${GPU_CASE_CLASS:?}" +: "${GPU_CASE_RANKS:?}" +: "${GPU_CASE_MANIFEST:?}" +: "${SLURM_ARRAY_JOB_ID:?}" +: "${SLURM_ARRAY_TASK_ID:?}" + +[[ "$SLURM_NTASKS" == "$GPU_CASE_RANKS" ]] +[[ "$SLURM_GPUS_ON_NODE" == "$GPU_CASE_RANKS" ]] + +line_number=$((SLURM_ARRAY_TASK_ID + 1)) +line=$(sed -n "${line_number}p" "$GPU_CASE_MANIFEST") +[[ -n $line ]] || { + echo "No manifest row $line_number in $GPU_CASE_MANIFEST" >&2 + exit 2 +} +IFS=$'\t' read -r suite case_name extra <<< "$line" +[[ -z ${extra:-} ]] +[[ $suite =~ ^[A-Za-z0-9_.-]+$ ]] +[[ $case_name =~ ^[A-Za-z0-9_.-]+$ ]] + +matrix_root="$RESULT_ROOT/case-matrix" +task_key="${GPU_CASE_CLASS}-${SLURM_ARRAY_TASK_ID}" +task_root="$matrix_root/tasks/$task_key" +work_root="$task_root/work" +status_root="$matrix_root/status" +status_file="$status_root/$task_key.tsv" +[[ ! -e "$task_root" ]] +mkdir -p "$work_root/tests/$suite" "$status_root" "$task_root/launcher" + +start_epoch=$(date +%s) +echo "SAI_GPU_CASE_START class=$GPU_CASE_CLASS task=$SLURM_ARRAY_TASK_ID suite=$suite case=$case_name ranks=$GPU_CASE_RANKS" + +ln -s "$CI_SOURCE/tests/integrate" "$work_root/tests/integrate" +ln -s "$CI_SOURCE/tests/PP_ORB" "$work_root/tests/PP_ORB" +rsync -a "$CI_SOURCE/tests/$suite/$case_name" "$work_root/tests/$suite/" +printf '%s\n' "$case_name" > "$work_root/tests/$suite/CASES.task.txt" + +source "$TOOLCHAIN_FILE" +declare -F sai_load_toolchain >/dev/null +sai_load_toolchain +source "/opt/sai_config/mps_mapping.d/${SLURM_JOB_PARTITION}.bash" +unset NCCL_IB_DISABLE + +SAI_SYSTEM_MPIRUN=$(command -v mpirun) +export SAI_SYSTEM_MPIRUN MAP_OPT +ln -s "$CONTROL_ROOT/mpirun_with_mapping.sh" "$task_root/launcher/mpirun" +export PATH="$task_root/launcher:$PATH" +export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" + +ABACUS="$INSTALL_ROOT/bin/abacus" +[[ -x "$ABACUS" ]] +ABACUS_LDD=$(ldd "$ABACUS") +if grep -q 'not found' <<< "$ABACUS_LDD"; then + echo "ABACUS has unresolved runtime dependencies" >&2 + exit 1 +fi +NCCL_LOADED=$(awk '$1 == "libnccl.so.2" {print $3; exit}' <<< "$ABACUS_LDD") +NCCL_EXPECTED=$(readlink -f "$SAI_NCCL_ROOT/lib/libnccl.so.2") +[[ $(readlink -f "$NCCL_LOADED") == "$NCCL_EXPECTED" ]] + +set +e +bash "$CONTROL_ROOT/run_gpu_case_attempts.sh" \ + "$work_root/tests/$suite" "$work_root/tests/$suite/$case_name" \ + "$case_name" "$ABACUS" "$GPU_CASE_RANKS" "$OMP_NUM_THREADS" \ + "$task_root" +test_rc=$? +set -e + +metadata="$task_root/pmix-retry.tsv" +if [[ ! -f $metadata ]]; then + state=INFRA + test_rc=2 +else + final_pmix=$(awk -F '\t' '$1 == "final_pmix" {print $2}' "$metadata") + [[ $final_pmix == 0 || $final_pmix == 1 ]] + case $test_rc in + 0) state=PASS ;; + 124|137|143) state=TIMEOUT ;; + *) + if [[ $final_pmix -eq 1 ]]; then + state=INFRA + else + state=FAIL + fi + ;; + esac +fi +elapsed=$(( $(date +%s) - start_epoch )) +status_tmp=$(mktemp "$status_root/.${task_key}.status.XXXXXX") +printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$GPU_CASE_CLASS" "$SLURM_ARRAY_TASK_ID" "$suite" "$case_name" \ + "$GPU_CASE_RANKS" "$state" "$test_rc" "$elapsed" "$SLURM_ARRAY_JOB_ID" \ + > "$status_tmp" +mv -T "$status_tmp" "$status_file" + +echo "SAI_GPU_CASE_RESULT class=$GPU_CASE_CLASS task=$SLURM_ARRAY_TASK_ID suite=$suite case=$case_name state=$state rc=$test_rc elapsed=$elapsed" +exit "$test_rc" diff --git a/ci/sai/tests/test_sai_ci.sh b/ci/sai/tests/test_sai_ci.sh new file mode 100755 index 00000000000..72ed1ad8631 --- /dev/null +++ b/ci/sai/tests/test_sai_ci.sh @@ -0,0 +1,1262 @@ +#!/usr/bin/env bash + +set -euo pipefail + +test_root=$(mktemp -d) +original_path=$PATH +tests_run=0 + +cleanup() { + local pid + for pid in $(jobs -pr); do + kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + done + rm -rf "$test_root" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_file() { + [[ -f $1 ]] || fail "missing file: $1" +} + +assert_not_exists() { + [[ ! -e $1 && ! -L $1 ]] || fail "unexpected path: $1" +} + +assert_contains() { + grep -Fq -- "$2" "$1" || fail "$1 does not contain: $2" +} + +assert_not_contains() { + if grep -Fq -- "$2" "$1"; then + fail "$1 unexpectedly contains: $2" + fi +} + +run_test() { + local name=$1 + shift + "$@" + tests_run=$((tests_run + 1)) + echo "PASS: $name" +} + +wait_for_file() { + local path=$1 + local _ + for _ in {1..100}; do + [[ -e $path ]] && return 0 + sleep 0.02 + done + fail "timed out waiting for $path" +} + +test_configure_ssh_client() { + local root=$test_root/ssh + local key=$root/input-key + local output=$root/client + local log=$root/output.log + mkdir -p "$root" + ssh-keygen -q -t ed25519 -N '' -C sai-ci-secret-marker -f "$key" + printf '[sai.example]:12022 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITest\n' \ + > "$root/known_hosts" + + SAI_SSH_PRIVATE_KEY=$(<"$key") \ + SAI_SSH_HOST=sai.example \ + SAI_SSH_PORT=12022 \ + SAI_SSH_USER=abacususer01 \ + bash ci/sai/configure_ssh_client.sh "$output" "$root/known_hosts" \ + > "$log" + + assert_file "$output/config" + assert_file "$output/id_ed25519" + [[ $(stat -c %a "$output/config") == 600 ]] + [[ $(stat -c %a "$output/id_ed25519") == 600 ]] + ssh-keygen -y -f "$output/id_ed25519" >/dev/null + assert_contains "$output/config" 'ClearAllForwardings yes' + assert_contains "$output/config" 'StrictHostKeyChecking yes' + assert_contains "$output/config" 'Compression yes' + assert_contains "$output/config" 'ConnectionAttempts 4' + assert_contains "$output/config" 'ConnectTimeout 30' + assert_contains "$output/config" 'ControlMaster auto' + assert_contains "$output/config" "ControlPath $output/control-%C" + assert_contains "$output/config" 'ControlPersist 15m' + assert_not_contains "$log" 'PRIVATE KEY' + assert_not_contains "$log" 'sai-ci-secret-marker' +} + +test_workflow_security_policy() { + local workflow=.github/workflows/sai-gpu-full.yml + local bootstrap=.github/workflows/sai-bootstrap.yml + local toolchain=ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example + assert_contains "$workflow" 'cron: "30 20 * * *"' + assert_contains "$workflow" "name: \${{ github.event_name == 'schedule' && 'sai-ssh-scheduled' || 'sai-ssh-manual' }}" + assert_contains "$workflow" "group: sai-gpu-\${{ github.event_name == 'schedule' && 'daily' || github.run_id }}" + assert_not_contains "$workflow" 'group: sai-gpu-rebuild' + assert_contains "$workflow" 'ref: ${{ github.event.repository.default_branch }}' + assert_contains "$workflow" 'Approved code SHA; executes as abacususer01 on SAI' + assert_contains "$bootstrap" 'name: sai-ssh-manual' + assert_contains "$bootstrap" 'ref: ${{ github.event.repository.default_branch }}' + assert_contains "$bootstrap" 'rsync -az -e "ssh -F $SAI_SSH_CONFIG"' + assert_contains "$workflow" 'ssh -F "$SAI_SSH_CONFIG" -o ConnectionAttempts=1 -MNf sai-ci' + assert_contains "$bootstrap" 'ssh -F "$SAI_SSH_CONFIG" -o ConnectionAttempts=1 -MNf sai-ci' + assert_contains "$workflow" 'ssh -F "$SAI_SSH_CONFIG" -O exit sai-ci' + assert_contains "$bootstrap" 'ssh -F "$SAI_SSH_CONFIG" -O exit sai-ci' + assert_contains ci/sai/run_remote_ci.sh \ + 'export SAI_CUSOLVERMP_ROOT=$SAI_NVIDIA_MP_ROOT/libcusolvermp-linux-x86_64-0.9.0.6427_cuda12-archive' + assert_contains ci/sai/run_remote_ci.sh \ + 'export SAI_CUBLASMP_ROOT=$SAI_NVIDIA_MP_ROOT/libcublasmp-linux-x86_64-0.9.1.3056_cuda12-archive' + assert_contains "$toolchain" 'module load nvhpc/26.3-gnu-cuda12-tuned' + assert_contains "$toolchain" 'export SAI_NCCL_ROOT=$NCCL_ROOT' + assert_not_contains "$toolchain" 'module load nccl/' + assert_not_contains "$toolchain" 'export SAI_NCCL_ROOT=/opt/' + assert_contains "$workflow" 'source_transfer_cache.sh' + assert_contains "$workflow" 'git -C source diff --binary --full-index --no-renames' + assert_contains "$workflow" 'empty_tree=$(git -C source hash-object -t tree /dev/null)' + assert_contains "$workflow" 'git -C source ls-tree -r -z --full-tree "$SOURCE_SHA"' + assert_contains "$workflow" '| gzip -1 > "$payload"' + assert_contains "$workflow" 'SOURCE_CACHE_BASE_SHA' + assert_contains "$workflow" 'run_namespace=${RUN_NAMESPACE_INPUT:-manual}' + assert_contains "$workflow" 'source_cache_role=baseline' + assert_contains "$workflow" 'source_cache_role=candidate' + assert_contains "$workflow" \ + 'runs/$RUN_NAMESPACE/$run_key' + assert_contains ci/sai/source_transfer_cache.sh 'flock 8' + assert_contains "$workflow" 'SOURCE_MANIFEST=$manifest' + assert_contains "$workflow" '"$SOURCE_PAYLOAD" "$SOURCE_MANIFEST"' + assert_contains "$workflow" '"sai-ci:$REMOTE_SOURCE_TRANSFER_ROOT/"' + assert_not_contains "$workflow" '"sai-ci:$REMOTE_RUN_ROOT/source/"' + assert_contains ci/sai/probe_remote_sai.sh 'rsync curl git gzip tar xz' + if sed -n '/^on:/,/^permissions:/p' "$workflow" | grep -Eq '^[[:space:]]+pull_request:'; then + fail 'GPU workflow must not run automatically for pull requests' + fi +} + +test_control_executable_modes() { + local path mode + for path in \ + ci/sai/build_gpu.sbatch \ + ci/sai/mpirun_with_mapping.sh \ + ci/sai/run_slurm_job.sh \ + ci/sai/test_gpu.sbatch \ + ci/sai/test_gpu_case.sh; do + mode=$(git ls-files -s -- "$path" | awk 'NR == 1 {print $1}') + [[ $mode == 100755 ]] || fail "$path has Git mode ${mode:-untracked}, expected 100755" + done +} + +test_pmix_startup_retry() { + local root=$test_root/pmix-retry + local suite=$root/work/tests/suite + local case_dir=$suite/case + local integrate=$root/work/tests/integrate + local results=$root/results + local abacus=$root/abacus + local fake_count=$root/count + local task + mkdir -p "$case_dir" "$integrate" "$results" "$root/bin" + : > "$abacus" + chmod +x "$abacus" + cat > "$root/bin/sleep" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + chmod +x "$root/bin/sleep" + cat > "$integrate/Autotest.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +count=0 +if [[ -f $FAKE_COUNT_FILE ]]; then + count=$(<"$FAKE_COUNT_FILE") +fi +count=$((count + 1)) +printf '%s\n' "$count" > "$FAKE_COUNT_FILE" +emit_pmix_failure() { + echo 'PMIX ERROR: PMIX_ERR_FILE_OPEN_FAILURE' + echo '*** An error occurred in MPI_Init_thread' + echo 'PMIx_Init failed for the following reason:' + exit 1 +} +case $FAKE_MODE in + pmix_once) + if [[ $count -eq 1 ]]; then + mkdir -p "$FAKE_CASE_DIR/OUT.autotest" + : > "$FAKE_CASE_DIR/log.txt" + emit_pmix_failure + fi + [[ ! -e $FAKE_CASE_DIR/OUT.autotest ]] + [[ ! -e $FAKE_CASE_DIR/log.txt ]] + echo 'recovered on the second attempt' + ;; + pmix_always) + emit_pmix_failure + ;; + pmix_timeout) + echo 'PMIX ERROR: PMIX_ERR_FILE_OPEN_FAILURE' + echo '*** An error occurred in MPI_Init_thread' + echo 'PMIx_Init failed for the following reason:' + exit 124 + ;; + normal_failure) + echo 'numerical comparison failed' + exit 1 + ;; + *) exit 2 ;; +esac +EOF + chmod +x "$integrate/Autotest.sh" + + task=$results/recovered + mkdir -p "$task" + FAKE_MODE=pmix_once FAKE_COUNT_FILE=$fake_count FAKE_CASE_DIR=$case_dir \ + RESULT_ROOT=$results PATH="$root/bin:$PATH" \ + bash ci/sai/run_gpu_case_attempts.sh \ + "$suite" "$case_dir" case "$abacus" 2 8 "$task" \ + > "$root/recovered.log" + [[ $(<"$fake_count") == 2 ]] + assert_contains "$task/pmix-retry.tsv" $'attempts\t2' + assert_contains "$task/pmix-retry.tsv" $'retried\t1' + assert_contains "$task/pmix-retry.tsv" $'final_pmix\t0' + assert_contains "$task/case.log" 'SAI_PMIX_STARTUP_RETRY' + assert_contains "$task/case-attempt-2.log" 'recovered on the second attempt' + + rm -f "$fake_count" + task=$results/persistent + mkdir -p "$task" + if FAKE_MODE=pmix_always FAKE_COUNT_FILE=$fake_count FAKE_CASE_DIR=$case_dir \ + RESULT_ROOT=$results PATH="$root/bin:$PATH" \ + bash ci/sai/run_gpu_case_attempts.sh \ + "$suite" "$case_dir" case "$abacus" 2 8 "$task" \ + > "$root/persistent.log"; then + fail 'persistent PMIx startup failure returned success' + fi + [[ $(<"$fake_count") == 2 ]] + assert_contains "$task/pmix-retry.tsv" $'attempts\t2' + assert_contains "$task/pmix-retry.tsv" $'final_pmix\t1' + + rm -f "$fake_count" + task=$results/normal-failure + mkdir -p "$task" + if FAKE_MODE=normal_failure FAKE_COUNT_FILE=$fake_count FAKE_CASE_DIR=$case_dir \ + RESULT_ROOT=$results PATH="$root/bin:$PATH" \ + bash ci/sai/run_gpu_case_attempts.sh \ + "$suite" "$case_dir" case "$abacus" 2 8 "$task" \ + > "$root/normal-failure.log"; then + fail 'normal case failure returned success' + fi + [[ $(<"$fake_count") == 1 ]] + assert_contains "$task/pmix-retry.tsv" $'attempts\t1' + assert_contains "$task/pmix-retry.tsv" $'retried\t0' + assert_contains "$task/pmix-retry.tsv" $'final_pmix\t0' + + rm -f "$fake_count" + task=$results/pmix-timeout + mkdir -p "$task" + if FAKE_MODE=pmix_timeout FAKE_COUNT_FILE=$fake_count FAKE_CASE_DIR=$case_dir \ + RESULT_ROOT=$results PATH="$root/bin:$PATH" \ + bash ci/sai/run_gpu_case_attempts.sh \ + "$suite" "$case_dir" case "$abacus" 2 8 "$task" \ + > "$root/pmix-timeout.log"; then + fail 'PMIx timeout returned success' + fi + [[ $(<"$fake_count") == 1 ]] + assert_contains "$task/pmix-retry.tsv" $'attempts\t1' + assert_contains "$task/pmix-retry.tsv" $'retried\t0' + assert_contains "$task/pmix-retry.tsv" $'final_pmix\t0' + assert_contains "$task/pmix-retry.tsv" $'final_rc\t124' +} + +test_gpu_matrix_submission_policy() { + local script=ci/sai/run_gpu_case_matrix.sh + local multinode=ci/sai/test_gpu.sbatch + local launcher=ci/sai/test_gpu_case.sh + local summary=ci/sai/summarize_gpu_case_matrix.sh + assert_contains "$script" \ + 'declare -A limits=([gpu1]=2 [gpu2]=8 [gpu4]=8)' + assert_contains "$script" 'export GPU_CASE_CLASS=$class' + assert_contains "$script" 'export GPU_CASE_RANKS=${ranks[$class]}' + assert_contains "$script" 'export GPU_CASE_MANIFEST=$manifest' + assert_contains "$script" '--export=ALL' + assert_not_contains "$script" '--export="ALL,' + assert_contains "$script" 'declare -A ranks=([gpu1]=1 [gpu2]=2 [gpu4]=4)' + assert_contains "$script" 'declare -A qos=([gpu1]=flood-1o2gpu [gpu2]=flood-1o2gpu [gpu4]=flood-gpu)' + assert_contains "$script" '--ntasks="${ranks[$class]}"' + assert_contains "$script" '--gpus-per-node="${ranks[$class]}"' + assert_contains "$script" '"$CONTROL_ROOT/test_gpu_case.sh"' + assert_not_contains "$script" '--cpus-per-task' + assert_contains "$multinode" 'prepare_cusolvermp_smoke.sh' + assert_not_contains "$multinode" 'CASES_CUSOLVERMP_16GPU.txt' + assert_contains "$multinode" '#SBATCH --nodes=2' + assert_contains "$multinode" '#SBATCH --ntasks=16' + assert_contains "$multinode" '#SBATCH --ntasks-per-node=8' + assert_contains "$multinode" '#SBATCH --gpus-per-node=8' + assert_contains "$multinode" '19_NO_Si48_CUSOLVERMP_TDDFT_GPU' + assert_not_contains "$multinode" 'Autotest.sh' + assert_contains "$launcher" 'run_gpu_case_attempts.sh' + assert_contains "$launcher" 'state=INFRA' + assert_contains "$summary" '^(PASS|FAIL|TIMEOUT|INFRA)$' +} + +test_prepare_cusolvermp_smoke() { + local root=$test_root/cusolvermp-smoke + local source=$root/source + local results=$root/results + local case_name=19_NO_Si48_CUSOLVERMP_TDDFT_GPU + local repository_case=tests/15_rtTDDFT_GPU/$case_name + local source_case=$source/tests/15_rtTDDFT_GPU/$case_name + local input=$source_case/INPUT + local staged=$results/cusolvermp-smoke/15_rtTDDFT_GPU/$case_name/INPUT + local name + mkdir -p "$(dirname "$source_case")" "$source/tests/PP_ORB" + cp -a "$repository_case" "$source_case" + printf 'must not be staged\n' > "$source_case/UNTRUSTED_EXTRA" + CI_SOURCE=$source RESULT_ROOT=$results \ + bash ci/sai/prepare_cusolvermp_smoke.sh > "$root/prepare.log" + assert_contains "$input" 'ks_solver cusolvermp' + assert_contains "$staged" 'ks_solver cusolvermp' + for name in INPUT KPT README STRU; do + cmp "$source_case/$name" \ + "$results/cusolvermp-smoke/15_rtTDDFT_GPU/$case_name/$name" + done + if [[ -e $results/cusolvermp-smoke/15_rtTDDFT_GPU/$case_name/UNTRUSTED_EXTRA ]]; then + fail 'cuSolverMp smoke staging copied an unvalidated extra file' + fi + + printf '%s\n' INPUT_PARAMETERS 'ks_solver elpa' > "$input" + if CI_SOURCE=$source RESULT_ROOT=$root/missing-results \ + bash ci/sai/prepare_cusolvermp_smoke.sh > /dev/null 2>&1; then + fail 'cuSolverMp smoke staging accepted a missing cusolvermp line' + fi + + printf '%s\n' INPUT_PARAMETERS 'ks_solver cusolvermp' 'ks_solver cusolvermp' > "$input" + if CI_SOURCE=$source RESULT_ROOT=$root/duplicate-results \ + bash ci/sai/prepare_cusolvermp_smoke.sh > /dev/null 2>&1; then + fail 'cuSolverMp smoke staging accepted duplicate cusolvermp lines' + fi + + cp "$repository_case/INPUT" "$input" + rm -f "$source_case/KPT" + ln -s "$PWD/$repository_case/KPT" "$source_case/KPT" + if CI_SOURCE=$source RESULT_ROOT=$root/symlink-results \ + bash ci/sai/prepare_cusolvermp_smoke.sh > /dev/null 2>&1; then + fail 'cuSolverMp smoke staging accepted a symlinked case file' + fi +} + +test_prepare_remote_run_paths() { + local root=$test_root/remote-paths + local home=$root/home + local outside=$root/outside + local project=$home/projects/abacus-ci + local output=$root/create.out + local config_home=$root/config-escape-home + local registry_home=$root/registry-leaf-home + mkdir -p "$home/projects" "$outside" + + HOME=$home bash ci/sai/prepare_remote_run.sh \ + "$project" pr-123 123-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > "$output" + assert_file "$project/runs/pr-123/123-1/.ci-created" + assert_contains "$output" "SAI_PROJECT_ROOT=$project" + assert_contains "$output" "RUN_ROOT=$project/runs/pr-123/123-1" + if HOME=$home bash ci/sai/prepare_remote_run.sh \ + "$project" pr-123 123-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > /dev/null 2>&1; then + fail 'remote run collision was accepted' + fi + if HOME=$home bash ci/sai/prepare_remote_run.sh \ + "$project" ../escape 128-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > /dev/null 2>&1; then + fail 'unsafe run namespace was accepted' + fi + ln -s "$outside" "$project/runs/linked" + if HOME=$home bash ci/sai/prepare_remote_run.sh \ + "$project" linked 129-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > /dev/null 2>&1; then + fail 'symlinked run namespace was accepted' + fi + assert_not_exists "$outside/129-1" + + ln -s "$outside" "$home/projects/escape" + if HOME=$home bash ci/sai/prepare_remote_run.sh \ + "$home/projects/escape/project" manual 124-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > /dev/null 2>&1; then + fail 'symlink escape was accepted' + fi + if HOME=$home bash ci/sai/prepare_remote_run.sh \ + "$home/projects/../outside" manual 125-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > /dev/null 2>&1; then + fail 'dot-dot path was accepted' + fi + + mkdir -p "$config_home" "$root/outside-config" + ln -s "$root/outside-config" "$config_home/.config" + if HOME=$config_home bash ci/sai/prepare_remote_run.sh \ + "$config_home/project" manual 126-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > /dev/null 2>&1; then + fail 'remote setup accepted a .config symlink escape' + fi + assert_not_exists "$root/outside-config/abacus-sai-ci" + + mkdir -p "$registry_home/.config/abacus-sai-ci" + printf 'registry-sentinel\n' > "$root/registry-target" + ln -s "$root/registry-target" \ + "$registry_home/.config/abacus-sai-ci/project-roots" + if HOME=$registry_home bash ci/sai/prepare_remote_run.sh \ + "$registry_home/project" manual 127-1 \ + 0123456789abcdef0123456789abcdef01234567 \ + 89abcdef0123456789abcdef0123456789abcdef \ + > /dev/null 2>&1; then + fail 'remote setup accepted a registry leaf symlink' + fi + [[ $(<"$root/registry-target") == registry-sentinel ]] +} + +test_source_snapshot_cache() { + local root=$test_root/source-cache + local home=$root/home + local project=$home/projects/abacus-ci + local repository=$root/repository + local sha1 sha2 + local run1=$project/runs/100-1 + local run2=$project/runs/101-1 + local run3=$project/runs/102-1 + local run4=$project/runs/103-1 + local run5=$project/runs/104-1 + local run6=$project/runs/105-1 + local run7=$project/runs/106-1 + local run8=$project/runs/107-1 + local run9=$project/runs/108-1 + local candidate=$project/runs/pr-7658/109-1 + local invalid_candidate=$project/runs/pr-invalid/110-1 + local output transfer snapshot snapshot_name inode_run inode_cache orphan + mkdir -p "$repository" + git -C "$repository" init -q + git -C "$repository" config user.email ci@example.invalid + git -C "$repository" config user.name ci + printf '*.bat text eol=crlf\n' > "$repository/.gitattributes" + printf 'canonical line\r\n' > "$repository/windows.bat" + printf 'unchanged\n' > "$repository/keep.txt" + printf 'remove later\n' > "$repository/delete.txt" + printf '\x00\x01base\xff' > "$repository/data.bin" + printf '#!/bin/sh\necho base\n' > "$repository/tool.sh" + chmod 0644 "$repository/tool.sh" + ln -s keep.txt "$repository/link" + git -C "$repository" add . + git -C "$repository" commit -qm base + sha1=$(git -C "$repository" rev-parse HEAD) + rm "$repository/windows.bat" + git -C "$repository" checkout -q -- windows.bat + + mkdir -p "$run1/source" "$run1/control" "$run1/build" \ + "$run1/install" "$run1/results" + touch "$run1/.ci-created" + + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run1" "$sha1" baseline) + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") 'SOURCE_CACHE_BASE_SHA=none' + [[ $transfer == "$project/cache/source-transfers/100-1" ]] + git -C "$repository" diff --binary --full-index --no-renames \ + "$(git -C "$repository" hash-object -t tree /dev/null)" "$sha1" \ + | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha1" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run1" "$transfer" full "$sha1" > "$root/receive1.log" + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run1" "$transfer" "$sha1" > "$root/finalize1.log" + assert_contains "$run1/source/keep.txt" 'unchanged' + assert_file "$project/cache/source-latest" + assert_contains "$project/cache/source-latest" "$sha1.100-1" + snapshot=$project/cache/source-snapshots/$sha1.100-1 + assert_file "$snapshot/keep.txt" + assert_file "$snapshot.manifest.gz" + git -C "$repository" show "$sha1:windows.bat" > "$root/windows.blob" + cmp "$root/windows.blob" "$snapshot/windows.bat" + if cmp -s "$repository/windows.bat" "$snapshot/windows.bat"; then + fail 'full source payload came from the CRLF checkout instead of Git blobs' + fi + inode_run=$(stat -c %i "$run1/source/keep.txt") + inode_cache=$(stat -c %i "$snapshot/keep.txt") + [[ $inode_run != "$inode_cache" ]] || fail 'run source shares cache inode' + + printf 'changed\n' > "$repository/keep.txt" + printf '\x00\x01target\xfe' > "$repository/data.bin" + rm "$repository/delete.txt" "$repository/link" + printf '#!/bin/sh\necho target\n' > "$repository/tool.sh" + chmod 0755 "$repository/tool.sh" + ln -s data.bin "$repository/link" + printf 'added\n' > "$repository/add.txt" + git -C "$repository" add -A + git -C "$repository" commit -qm target + sha2=$(git -C "$repository" rev-parse HEAD) + + mkdir -p "$candidate/source" "$candidate/control" "$candidate/build" \ + "$candidate/install" "$candidate/results" + touch "$candidate/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$candidate" "$sha2" candidate) + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") "SOURCE_CACHE_BASE_SHA=$sha1" + assert_contains <(printf '%s\n' "$output") 'SOURCE_CACHE_ROLE=candidate' + git -C "$repository" diff --binary --full-index --no-renames \ + "$sha1" "$sha2" | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$candidate" "$transfer" delta "$sha2" > /dev/null + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$candidate" "$transfer" "$sha2" \ + > "$root/finalize-candidate.log" + assert_contains "$candidate/source/keep.txt" 'changed' + assert_contains "$root/finalize-candidate.log" \ + "SOURCE_CACHE_PROMOTION=skipped role=candidate source_sha=$sha2" + assert_contains "$project/cache/source-latest" "$sha1.100-1" + assert_file "$snapshot/keep.txt" + assert_not_exists "$transfer" + + mkdir -p "$run2/source" "$run2/control" "$run2/build" \ + "$run2/install" "$run2/results" + touch "$run2/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run2" "$sha2" baseline) + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") "SOURCE_CACHE_BASE_SHA=$sha1" + assert_contains "$transfer/source/keep.txt" 'unchanged' + assert_file "$transfer/source/delete.txt" + git -C "$repository" diff --binary --full-index --no-renames \ + "$sha1" "$sha2" | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run2" "$transfer" delta "$sha2" > "$root/receive2.log" + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run2" "$transfer" "$sha2" > "$root/finalize2.log" + assert_contains "$run2/source/keep.txt" 'changed' + assert_contains "$run2/source/add.txt" 'added' + assert_not_exists "$run2/source/delete.txt" + [[ -x $run2/source/tool.sh ]] + [[ $(readlink "$run2/source/link") == data.bin ]] + cmp "$repository/data.bin" "$run2/source/data.bin" + assert_contains "$project/cache/source-latest" "$sha2.101-1" + assert_not_exists "$snapshot" + assert_not_exists "$snapshot.manifest.gz" + + orphan="$project/cache/source-snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.999-1" + mkdir -p "$orphan" + printf 'orphan\n' > "$orphan/file" + printf 'orphan manifest\n' | gzip -1 > "$orphan.manifest.gz" + mkdir -p "$run3/source" "$run3/control" "$run3/build" \ + "$run3/install" "$run3/results" + touch "$run3/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run3" "$sha2" baseline) + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") "SOURCE_CACHE_BASE_SHA=$sha2" + assert_not_exists "$orphan" + assert_not_exists "$orphan.manifest.gz" + git -C "$repository" diff --binary --full-index --no-renames \ + "$sha2" "$sha2" | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run3" "$transfer" delta "$sha2" > /dev/null + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run3" "$transfer" "$sha2" > /dev/null + + snapshot_name=$(<"$project/cache/source-latest") + snapshot="$project/cache/source-snapshots/$snapshot_name" + printf 'cache drift\n' > "$snapshot/keep.txt" + + mkdir -p "$invalid_candidate/source" "$invalid_candidate/control" \ + "$invalid_candidate/build" "$invalid_candidate/install" \ + "$invalid_candidate/results" + touch "$invalid_candidate/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$invalid_candidate" "$sha2" candidate \ + 2> "$root/candidate-drift.err") + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") 'SOURCE_CACHE_BASE_SHA=none' + assert_contains "$root/candidate-drift.err" \ + 'content_or_manifest_mismatch role=candidate' + assert_contains "$project/cache/source-latest" "$snapshot_name" + assert_not_exists "$project/cache/.source-latest.invalid.110-1" + assert_contains "$snapshot/keep.txt" 'cache drift' + git -C "$repository" diff --binary --full-index --no-renames \ + "$(git -C "$repository" hash-object -t tree /dev/null)" "$sha2" \ + | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$invalid_candidate" "$transfer" full "$sha2" > /dev/null + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$invalid_candidate" "$transfer" "$sha2" > /dev/null + assert_contains "$invalid_candidate/source/keep.txt" 'changed' + assert_contains "$project/cache/source-latest" "$snapshot_name" + assert_contains "$snapshot/keep.txt" 'cache drift' + + mkdir -p "$run4/source" "$run4/control" "$run4/build" \ + "$run4/install" "$run4/results" + touch "$run4/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run4" "$sha2" baseline 2> "$root/drift.err") + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") 'SOURCE_CACHE_BASE_SHA=none' + assert_contains "$root/drift.err" 'content_or_manifest_mismatch' + assert_file "$project/cache/.source-latest.invalid.103-1" + [[ -z $(find "$transfer/source" -mindepth 1 -print -quit) ]] + git -C "$repository" diff --binary --full-index --no-renames \ + "$(git -C "$repository" hash-object -t tree /dev/null)" "$sha2" \ + | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run4" "$transfer" full "$sha2" > /dev/null + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run4" "$transfer" "$sha2" > /dev/null + + snapshot_name=$(<"$project/cache/source-latest") + snapshot="$project/cache/source-snapshots/$snapshot_name" + printf 'extra\n' > "$snapshot/untracked.txt" + mkdir -p "$run5/source" "$run5/control" "$run5/build" \ + "$run5/install" "$run5/results" + touch "$run5/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run5" "$sha2" baseline 2> "$root/extra.err") + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") 'SOURCE_CACHE_BASE_SHA=none' + assert_contains "$root/extra.err" 'content_or_manifest_mismatch' + git -C "$repository" diff --binary --full-index --no-renames \ + "$(git -C "$repository" hash-object -t tree /dev/null)" "$sha2" \ + | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run5" "$transfer" full "$sha2" > /dev/null + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run5" "$transfer" "$sha2" > /dev/null + + printf 'not-a-cache-pointer\n' > "$project/cache/source-latest" + mkdir -p "$run6/source" "$run6/control" "$run6/build" \ + "$run6/install" "$run6/results" + touch "$run6/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run6" "$sha2" baseline \ + 2> "$root/malformed-pointer.err") + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") 'SOURCE_CACHE_BASE_SHA=none' + assert_contains "$root/malformed-pointer.err" 'malformed_pointer' + git -C "$repository" diff --binary --full-index --no-renames \ + "$(git -C "$repository" hash-object -t tree /dev/null)" "$sha2" \ + | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run6" "$transfer" full "$sha2" > /dev/null + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run6" "$transfer" "$sha2" > /dev/null + + printf '%s\n' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.777-1 \ + > "$project/cache/source-latest" + mkdir -p "$run7/source" "$run7/control" "$run7/build" \ + "$run7/install" "$run7/results" + touch "$run7/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run7" "$sha2" baseline \ + 2> "$root/dangling-pointer.err") + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + assert_contains <(printf '%s\n' "$output") 'SOURCE_CACHE_BASE_SHA=none' + assert_contains "$root/dangling-pointer.err" 'content_or_manifest_mismatch' + git -C "$repository" diff --binary --full-index --no-renames \ + "$(git -C "$repository" hash-object -t tree /dev/null)" "$sha2" \ + | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run7" "$transfer" full "$sha2" > /dev/null + HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run7" "$transfer" "$sha2" > /dev/null + + mkdir -p "$run8/source" "$run8/control" "$run8/build" \ + "$run8/install" "$run8/results" + touch "$run8/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run8" "$sha2" baseline) + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | gzip -1 > "$transfer/source-manifest.gz" + printf 'not gzip\n' > "$transfer/source-payload.gz" + if HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run8" "$transfer" delta "$sha2" > /dev/null 2>&1; then + fail 'source transfer accepted a malformed gzip payload' + fi + printf 'not a Git patch\n' | gzip -1 > "$transfer/source-payload.gz" + if HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run8" "$transfer" delta "$sha2" > /dev/null 2>&1; then + fail 'source transfer accepted an invalid Git patch' + fi + + mkdir -p "$run9/source" "$run9/control" "$run9/build" \ + "$run9/install" "$run9/results" + touch "$run9/.ci-created" + output=$(HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run9" "$sha2" baseline) + transfer=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") + rm "$transfer/source/windows.bat" + git -C "$repository" diff --binary --full-index --no-renames \ + "$sha2" "$sha2" | gzip -1 > "$transfer/source-payload.gz" + git -C "$repository" ls-tree -r -z --full-tree "$sha2" \ + | head -c -1 | gzip -1 > "$transfer/source-manifest.gz" + HOME=$home bash ci/sai/source_transfer_cache.sh receive \ + "$project" "$run9" "$transfer" delta "$sha2" > /dev/null + if HOME=$home bash ci/sai/source_transfer_cache.sh finalize \ + "$project" "$run9" "$transfer" "$sha2" > /dev/null 2>&1; then + fail 'source transfer accepted a manifest with an unterminated record' + fi + + if HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run2" "$sha2" > /dev/null 2>&1; then + fail 'source transfer accepted an omitted cache role' + fi + if HOME=$home bash ci/sai/source_transfer_cache.sh prepare \ + "$project" "$run2" "$sha2" baseline > /dev/null 2>&1; then + fail 'source transfer accepted a reused run key' + fi +} + +test_prepare_cleanup_install() { + local root=$test_root/cleanup-staging + local home=$root/home + local outside=$root/outside + local project=$home/project + local output=$root/prepare.out + local registry_home=$root/registry-home + mkdir -p "$home" "$outside" + + HOME=$home bash ci/sai/prepare_cleanup_install.sh "$project" 300-1 \ + > "$output" + assert_file "$project/diagnostics/cleanup-300-1/cleanup_sai_runs.sh" + assert_file "$project/diagnostics/cleanup-300-1/.ci-diagnostic" + assert_contains "$output" "SAI_PROJECT_ROOT=$project" + assert_contains "$output" \ + "CLEANUP_STAGING_FILE=$project/diagnostics/cleanup-300-1/cleanup_sai_runs.sh" + assert_contains "$home/.config/abacus-sai-ci/project-roots" "$project" + if HOME=$home bash ci/sai/prepare_cleanup_install.sh \ + "$project" 300-1 > /dev/null 2>&1; then + fail 'cleanup staging collision was accepted' + fi + + ln -s "$outside" "$home/escape" + if HOME=$home bash ci/sai/prepare_cleanup_install.sh \ + "$home/escape/project" 301-1 > /dev/null 2>&1; then + fail 'cleanup staging symlink escape was accepted' + fi + + mkdir -p "$registry_home/.config/abacus-sai-ci" + printf 'registry-sentinel\n' > "$root/registry-target" + ln -s "$root/registry-target" \ + "$registry_home/.config/abacus-sai-ci/project-roots" + if HOME=$registry_home bash ci/sai/prepare_cleanup_install.sh \ + "$registry_home/project" 302-1 > /dev/null 2>&1; then + fail 'cleanup staging accepted a registry leaf symlink' + fi + [[ $(<"$root/registry-target") == registry-sentinel ]] +} + +make_cached_archive() { + local destination=$1 + local sha=$2 + local header=$3 + local library=$4 + mkdir -p "$destination/include" "$destination/lib" + : > "$destination/include/$header" + : > "$destination/lib/$library" + printf '%s\n' "$sha" > "$destination/.archive-sha256" +} + +test_nvidia_archive_cache() { + local root=$test_root/archive-cache + local project=$root/project + local vendor=$project/vendor/nvidia-mp-0.9-archive + local cusolver=$vendor/libcusolvermp-linux-x86_64-0.9.0.6427_cuda12-archive + local cublas=$vendor/libcublasmp-linux-x86_64-0.9.1.3056_cuda12-archive + local fake_bin=$root/bin + local curl_called=$root/curl-called + local tarlink_project=$root/tarlink-project + local tarlink_downloads=$tarlink_project/vendor/downloads + local tarlink_outside=$root/tarlink-outside + mkdir -p "$fake_bin" "$project/vendor/downloads" + printf 'lock-sentinel\n' > "$root/lock-target" + ln -s "$root/lock-target" \ + "$project/vendor/downloads/.nvidia-mp-download.lock" + make_cached_archive "$cusolver" \ + 3b071ce69c6a6a6bb7add8784e6a3fc54e9a64a8f2c1c7da40b03bcde39eb57c \ + cusolverMp.h libcusolverMp.so.0 + make_cached_archive "$cublas" \ + 35fea4df2bb08a496981f34c0d486f0753d3766a31d60dbe6daa6f16673cd1cc \ + cublasmp.h libcublasmp.so.0 + cat > "$fake_bin/curl" < "$root/reuse.log" + assert_not_exists "$curl_called" + assert_contains "$root/reuse.log" 'NVIDIA_MP_ARCHIVES_READY' + [[ $(<"$root/lock-target") == lock-sentinel ]] + + printf '%s\n' bad-sha > "$cublas/.archive-sha256" + if PATH="$fake_bin:$original_path" HOME=$root TMPDIR=$root/missing-tmp \ + SAI_PROJECT_ROOT=$project \ + bash ci/sai/prepare_nvidia_mp.sh > /dev/null 2> "$root/mismatch.err"; then + fail 'mismatched extracted archive cache was accepted' + fi + assert_contains "$root/mismatch.err" 'cache' + assert_not_exists "$curl_called" + + mkdir -p "$tarlink_downloads" "$tarlink_outside" + ln -s "$tarlink_outside" \ + "$tarlink_downloads/libcusolvermp-linux-x86_64-0.9.0.6427_cuda12-archive.tar.xz" + if PATH="$fake_bin:$original_path" HOME=$root TMPDIR=$root/missing-tmp \ + SAI_PROJECT_ROOT=$tarlink_project \ + bash ci/sai/prepare_nvidia_mp.sh > /dev/null 2> "$root/tarlink.err"; then + fail 'tarball cache symlink to an outside directory was accepted' + fi + assert_contains "$root/tarlink.err" 'tarball cache' + assert_not_exists "$curl_called" + [[ -z $(find "$tarlink_outside" -mindepth 1 -print -quit) ]] + + mkdir -p "$root/escaped-project" "$root/outside-vendor" + ln -s "$root/outside-vendor" "$root/escaped-project/vendor" + if PATH="$fake_bin:$original_path" HOME=$root TMPDIR=$root/missing-tmp \ + SAI_PROJECT_ROOT=$root/escaped-project \ + bash ci/sai/prepare_nvidia_mp.sh > /dev/null 2> "$root/escape.err"; then + fail 'vendor symlink escape was accepted' + fi + assert_not_exists "$curl_called" +} + +test_artifact_collection() { + local root=$test_root/artifacts + local home=$root/home + local run=$home/project/runs/200-1 + local empty_extract=$root/empty + local full_extract=$root/full + mkdir -p "$run/results" "$empty_extract" "$full_extract" + : > "$run/.ci-created" + + HOME=$home TMPDIR=$root/missing-tmp \ + bash ci/sai/collect_remote_artifacts.sh "$run" \ + | tar -xzf - -C "$empty_extract" + assert_file "$empty_extract/.ci-created" + + mkdir -p "$run/results/case-matrix" \ + "$run/source/tests/11_GPU/case/OUT.test" + printf 'summary\n' > "$run/results/case-matrix/summary.md" + printf 'ignore\n' > "$run/results/case-matrix/raw.bin" + printf 'result\n' > "$run/source/tests/11_GPU/case/result.out" + printf 'running\n' > "$run/source/tests/11_GPU/case/OUT.test/running.log" + HOME=$home TMPDIR=$root/missing-tmp \ + bash ci/sai/collect_remote_artifacts.sh "$run" \ + | tar -xzf - -C "$full_extract" + assert_file "$full_extract/results/case-matrix/summary.md" + assert_file "$full_extract/source/tests/11_GPU/case/result.out" + assert_file "$full_extract/source/tests/11_GPU/case/OUT.test/running.log" + assert_not_exists "$full_extract/results/case-matrix/raw.bin" + + mkdir -p "$run/build" "$root/outside" + printf 'outside\n' > "$root/outside/toolchain-summary.txt" + ln -s "$root/outside/toolchain-summary.txt" \ + "$run/build/toolchain-summary.txt" + if HOME=$home TMPDIR=$root/missing-tmp \ + bash ci/sai/collect_remote_artifacts.sh "$run" \ + > "$root/escaped.tar.gz" 2> "$root/escape.err"; then + fail 'artifact collector accepted a symlink escape' + fi +} + +make_cleanup_fixture() { + local home=$1 + local project=$home/project + mkdir -p "$project/runs" "$project/diagnostics" \ + "$project/cache/source-transfers/206-1" \ + "$project/cache/source-transfers/207-1" \ + "$home/.config/abacus-sai-ci" + printf '%s\n' "$project" > "$home/.config/abacus-sai-ci/project-roots" + + mkdir -p "$project/runs/201-1/results" + : > "$project/runs/201-1/.artifacts-uploaded" + touch -d '73 hours ago' "$project/runs/201-1/.artifacts-uploaded" + + mkdir -p "$project/runs/202-1/results" + : > "$project/runs/202-1/.artifacts-uploaded" + touch -d '71 hours ago' "$project/runs/202-1/.artifacts-uploaded" + + mkdir -p "$project/runs/203-1/results" + : > "$project/runs/203-1/.ci-created" + touch -d '169 hours ago' "$project/runs/203-1/.ci-created" + + mkdir -p "$project/runs/204-1/results" + : > "$project/runs/204-1/.artifacts-uploaded" + touch -d '73 hours ago' "$project/runs/204-1/.artifacts-uploaded" + printf 'SLURM_JOB_ID=701\n' > "$project/runs/204-1/results/build-submit.log" + + mkdir -p "$project/runs/205-1/results" + : > "$project/runs/205-1/.artifacts-uploaded" + touch -d '73 hours ago' "$project/runs/205-1/.artifacts-uploaded" + printf 'SLURM_JOB_ID=702\n' > "$project/runs/205-1/results/build-submit.log" + + mkdir -p "$project/runs/pr-7658/208-1/results" + : > "$project/runs/pr-7658/208-1/.artifacts-uploaded" + touch -d '73 hours ago' \ + "$project/runs/pr-7658/208-1/.artifacts-uploaded" + + mkdir -p "$project/diagnostics/diagnostic-old" + : > "$project/diagnostics/diagnostic-old/.ci-diagnostic" + touch -d '169 hours ago' "$project/diagnostics/diagnostic-old/.ci-diagnostic" + + : > "$project/cache/source-transfers/206-1/.ci-source-transfer" + touch -d '169 hours ago' \ + "$project/cache/source-transfers/206-1/.ci-source-transfer" + : > "$project/cache/source-transfers/207-1/.ci-source-transfer" + touch -d '167 hours ago' \ + "$project/cache/source-transfers/207-1/.ci-source-transfer" + + diagnostic_only=$home/diagnostic-only + mkdir -p "$diagnostic_only/diagnostics/diagnostic-old" + : > "$diagnostic_only/diagnostics/diagnostic-old/.ci-diagnostic" + touch -d '169 hours ago' \ + "$diagnostic_only/diagnostics/diagnostic-old/.ci-diagnostic" + printf '%s\n' "$diagnostic_only" >> \ + "$home/.config/abacus-sai-ci/project-roots" +} + +test_cleanup_retention() { + local root=$test_root/cleanup + local home=$root/home + local project=$home/project + local fake_bin=$root/bin + local squeue_mode=$root/squeue-mode + local squeue_log=$root/squeue.log + local escape_home=$root/escape-home + local leaf_home=$root/leaf-home + mkdir -p "$fake_bin" + make_cleanup_fixture "$home" + cat > "$fake_bin/squeue" <> "$squeue_log" +[[ " \$* " == *' --noheader '* ]] +[[ " \$* " == *' --user='* ]] +[[ " \$* " == *' --format=%F '* ]] +if [[ -f "$squeue_mode" ]]; then + exit 1 +fi +echo 701 +EOF + chmod +x "$fake_bin/squeue" + + PATH="$fake_bin:$original_path" HOME=$home \ + bash ci/sai/cleanup_sai_runs.sh --dry-run > "$root/dry-run.log" + assert_contains "$root/dry-run.log" "DRY_RUN delete path=$project/runs/201-1" + assert_contains "$root/dry-run.log" "DRY_RUN delete path=$project/runs/203-1" + assert_contains "$root/dry-run.log" "DRY_RUN delete path=$project/runs/205-1" + assert_contains "$root/dry-run.log" \ + "DRY_RUN delete path=$project/runs/pr-7658/208-1" + assert_contains "$root/dry-run.log" "DRY_RUN delete path=$project/diagnostics/diagnostic-old" + assert_contains "$root/dry-run.log" "DRY_RUN delete path=$project/cache/source-transfers/206-1" + assert_contains "$root/dry-run.log" "DRY_RUN delete path=$home/diagnostic-only/diagnostics/diagnostic-old" + assert_contains "$root/dry-run.log" "SKIP active_or_unknown path=$project/runs/204-1" + assert_file "$project/runs/201-1/.artifacts-uploaded" + [[ -d $project/runs/pr-7658 ]] + + : > "$squeue_mode" + PATH="$fake_bin:$original_path" HOME=$home \ + bash ci/sai/cleanup_sai_runs.sh > "$root/cleanup.log" + assert_not_exists "$project/runs/201-1" + assert_not_exists "$project/runs/203-1" + assert_not_exists "$project/runs/pr-7658/208-1" + [[ -d $project/runs/pr-7658 ]] + assert_not_exists "$project/diagnostics/diagnostic-old" + assert_not_exists "$project/cache/source-transfers/206-1" + assert_not_exists "$home/diagnostic-only/diagnostics/diagnostic-old" + assert_file "$project/runs/202-1/.artifacts-uploaded" + assert_file "$project/runs/204-1/.artifacts-uploaded" + assert_file "$project/runs/205-1/.artifacts-uploaded" + assert_file "$project/cache/source-transfers/207-1/.ci-source-transfer" + + mkdir -p "$escape_home" "$root/outside-cache" + ln -s "$root/outside-cache" "$escape_home/.cache" + if PATH="$fake_bin:$original_path" HOME=$escape_home \ + bash ci/sai/cleanup_sai_runs.sh --dry-run > /dev/null 2>&1; then + fail 'cleanup accepted a .cache symlink escape' + fi + assert_not_exists "$root/outside-cache/abacus-sai-ci" + + mkdir -p "$leaf_home/.cache/abacus-sai-ci" \ + "$leaf_home/.local/state/abacus-sai-ci" + printf 'leaf-sentinel\n' > "$root/leaf-target" + ln -s "$root/leaf-target" "$leaf_home/.cache/abacus-sai-ci/cleanup.lock" + PATH="$fake_bin:$original_path" HOME=$leaf_home \ + bash ci/sai/cleanup_sai_runs.sh --dry-run > /dev/null + [[ $(<"$root/leaf-target") == leaf-sentinel ]] + ln -s "$root/leaf-target" "$leaf_home/.local/state/abacus-sai-ci/cleanup.log" + if PATH="$fake_bin:$original_path" HOME=$leaf_home \ + bash ci/sai/cleanup_sai_runs.sh --cron > /dev/null 2>&1; then + fail 'cleanup accepted a log leaf symlink' + fi + [[ $(<"$root/leaf-target") == leaf-sentinel ]] +} + +test_cleanup_cron_installation() { + local root=$test_root/cleanup-cron + local home=$root/home + local project=$home/project + local fake_bin=$root/bin + local crontab_file=$root/crontab + local original=$root/original-crontab + local crontab_failure=$root/crontab-failure + local staging_root=$project/diagnostics/cleanup-999-1 + local staged_cleanup=$staging_root/cleanup_sai_runs.sh + local escape_home=$root/escape-home + local escape_project=$escape_home/project + local escape_staging_root=$escape_project/diagnostics/cleanup-998-1 + local escape_staged=$escape_staging_root/cleanup_sai_runs.sh + local missing_tmp=$root/missing-tmp + local leaf_home=$root/leaf-home + local leaf_project=$leaf_home/project + local leaf_staging_root=$leaf_project/diagnostics/cleanup-997-1 + local leaf_staged=$leaf_staging_root/cleanup_sai_runs.sh + mkdir -p "$fake_bin" "$project" + stage_cleanup() { + mkdir -p "$staging_root" + : > "$staging_root/.ci-diagnostic" + cp ci/sai/cleanup_sai_runs.sh "$staged_cleanup" + } + cat > "$fake_bin/crontab" <&2 + exit 2 + elif [[ -f "$crontab_file" ]]; then + cat "$crontab_file" + else + echo "no crontab for \${USER:-unknown}" >&2 + exit 1 + fi +else + cp "\$1" "$crontab_file" +fi +EOF + chmod +x "$fake_bin/crontab" + + stage_cleanup + PATH="$fake_bin:$original_path" HOME=$home TMPDIR=$missing_tmp \ + bash ci/sai/install_cleanup_cron.sh "$project" "$staged_cleanup" \ + > "$root/first-install.log" + assert_contains "$crontab_file" '15 7 * * * $HOME/.local/libexec/abacus-sai-ci/cleanup_sai_runs.sh --cron' + + printf '%s\n' '5 1 * * * existing-job' > "$crontab_file" + cp "$crontab_file" "$original" + : > "$crontab_failure" + stage_cleanup + if PATH="$fake_bin:$original_path" HOME=$home TMPDIR=$missing_tmp \ + bash ci/sai/install_cleanup_cron.sh "$project" "$staged_cleanup" \ + > /dev/null 2> "$root/read-failure.err"; then + fail 'cleanup installer replaced crontab after a read failure' + fi + cmp "$original" "$crontab_file" + rm -f "$crontab_failure" + + printf '%s\n' '# BEGIN ABACUS_SAI_CI_CLEANUP' '5 1 * * * existing-job' \ + > "$crontab_file" + cp "$crontab_file" "$original" + stage_cleanup + if PATH="$fake_bin:$original_path" HOME=$home TMPDIR=$missing_tmp \ + bash ci/sai/install_cleanup_cron.sh "$project" "$staged_cleanup" \ + > /dev/null 2> "$root/invalid.err"; then + fail 'cleanup installer accepted an unmatched cron marker' + fi + cmp "$original" "$crontab_file" + + printf '%s\n' \ + 'MAILTO=owner@example.invalid' \ + '# BEGIN ABACUS_SAI_CI_CLEANUP' \ + '0 0 * * * obsolete-cleanup' \ + '# END ABACUS_SAI_CI_CLEANUP' \ + '5 1 * * * existing-job' > "$crontab_file" + PATH="$fake_bin:$original_path" HOME=$home TMPDIR=$missing_tmp \ + bash ci/sai/install_cleanup_cron.sh "$project" "$staged_cleanup" \ + > "$root/install.log" + assert_contains "$crontab_file" 'MAILTO=owner@example.invalid' + assert_contains "$crontab_file" '5 1 * * * existing-job' + assert_contains "$crontab_file" '15 7 * * * $HOME/.local/libexec/abacus-sai-ci/cleanup_sai_runs.sh --cron' + [[ $(grep -Fc '# BEGIN ABACUS_SAI_CI_CLEANUP' "$crontab_file") -eq 1 ]] + [[ $(grep -Fc '# END ABACUS_SAI_CI_CLEANUP' "$crontab_file") -eq 1 ]] + assert_contains "$home/.config/abacus-sai-ci/project-roots" "$project" + assert_not_contains "$root/install.log" 'MAILTO=owner@example.invalid' + assert_not_contains "$root/install.log" '5 1 * * * existing-job' + + mkdir -p "$escape_home" "$escape_project" "$root/outside-local" + ln -s "$root/outside-local" "$escape_home/.local" + mkdir -p "$escape_staging_root" + : > "$escape_staging_root/.ci-diagnostic" + cp ci/sai/cleanup_sai_runs.sh "$escape_staged" + if PATH="$fake_bin:$original_path" HOME=$escape_home TMPDIR=$missing_tmp \ + bash ci/sai/install_cleanup_cron.sh \ + "$escape_project" "$escape_staged" > /dev/null 2>&1; then + fail 'cleanup installer accepted a .local symlink escape' + fi + assert_not_exists "$root/outside-local/state" + + mkdir -p "$leaf_staging_root" \ + "$leaf_home/.local/libexec/abacus-sai-ci" + : > "$leaf_staging_root/.ci-diagnostic" + cp ci/sai/cleanup_sai_runs.sh "$leaf_staged" + printf 'install-sentinel\n' > "$root/install-target" + ln -s "$root/install-target" \ + "$leaf_home/.local/libexec/abacus-sai-ci/cleanup_sai_runs.sh" + if PATH="$fake_bin:$original_path" HOME=$leaf_home TMPDIR=$missing_tmp \ + bash ci/sai/install_cleanup_cron.sh \ + "$leaf_project" "$leaf_staged" > /dev/null 2>&1; then + fail 'cleanup installer accepted a target leaf symlink' + fi + [[ $(<"$root/install-target") == install-sentinel ]] +} + +test_mark_artifacts_uploaded() { + local root=$test_root/mark-uploaded + local home=$root/home + local run=$home/project/runs/400-1 + mkdir -p "$run" + printf 'upload-sentinel\n' > "$root/upload-target" + ln -s "$root/upload-target" "$run/.artifacts-uploaded" + + HOME=$home TMPDIR=$root/missing-tmp \ + bash ci/sai/mark_artifacts_uploaded.sh "$run" + [[ ! -L $run/.artifacts-uploaded ]] + assert_contains "$run/.artifacts-uploaded" 'uploaded_epoch=' + [[ $(<"$root/upload-target") == upload-sentinel ]] +} + +test_slurm_signal_cancellation() { + local root=$test_root/slurm-signal + local fake_bin=$root/bin + local submit_log=$root/submit.log + local output_pattern=$root/job-%j.out + local cancel_log=$root/scancel.log + local pid rc + mkdir -p "$fake_bin" "$root/source" + : > "$root/job.sbatch" + cat > "$fake_bin/sbatch" <<'EOF' +#!/usr/bin/env bash +echo 801 +EOF + cat > "$fake_bin/sacct" <<'EOF' +#!/usr/bin/env bash +echo '801 RUNNING 0:0' +EOF + cat > "$fake_bin/scancel" <> "$cancel_log" +EOF + chmod +x "$fake_bin/sbatch" "$fake_bin/sacct" "$fake_bin/scancel" + + PATH="$fake_bin:$original_path" HOME=$test_root TMPDIR=$root/missing-tmp \ + CI_SOURCE=$root/source \ + GITHUB_RUN_ID=1 GITHUB_RUN_ATTEMPT=1 \ + bash ci/sai/run_slurm_job.sh "$submit_log" "$output_pattern" \ + "$root/job.sbatch" > "$root/driver.log" 2>&1 & + pid=$! + wait_for_file "$submit_log" + kill -TERM "$pid" + set +e + wait "$pid" + rc=$? + set -e + [[ $rc -eq 143 ]] || fail "TERM returned $rc instead of 143" + assert_contains "$cancel_log" '801' +} + +test_slurm_launch_window_cancellation() { + local root=$test_root/slurm-launch-signal + local fake_bin=$root/bin + local submit_log=$root/submit.log + local output_pattern=$root/job-%j.out + local cancel_log=$root/scancel.log + local ready=$root/sbatch-ready + local pid rc + mkdir -p "$fake_bin" "$root/source" + : > "$root/job.sbatch" + cat > "$fake_bin/sbatch" < "$fake_bin/scancel" <> "$cancel_log" +EOF + chmod +x "$fake_bin/sbatch" "$fake_bin/scancel" + + PATH="$fake_bin:$original_path" HOME=$test_root TMPDIR=$root/missing-tmp \ + CI_SOURCE=$root/source \ + GITHUB_RUN_ID=2 GITHUB_RUN_ATTEMPT=1 \ + bash ci/sai/run_slurm_job.sh "$submit_log" "$output_pattern" \ + "$root/job.sbatch" > "$root/driver.log" 2>&1 & + pid=$! + wait_for_file "$ready" + kill -HUP "$pid" + set +e + wait "$pid" + rc=$? + set -e + [[ $rc -eq 143 ]] || fail "launch-window HUP returned $rc instead of 143" + assert_contains "$cancel_log" '802' +} + +run_test 'SSH client configuration' test_configure_ssh_client +run_test 'workflow security policy' test_workflow_security_policy +run_test 'control executable modes' test_control_executable_modes +run_test 'PMIx startup retry policy' test_pmix_startup_retry +run_test 'GPU matrix submission policy' test_gpu_matrix_submission_policy +run_test 'cuSolverMp smoke staging' test_prepare_cusolvermp_smoke +run_test 'remote path containment and collision' test_prepare_remote_run_paths +run_test 'compressed source snapshot cache' test_source_snapshot_cache +run_test 'cleanup staging containment and collision' test_prepare_cleanup_install +run_test 'NVIDIA archive cache reuse and rejection' test_nvidia_archive_cache +run_test 'artifact collection whitelist' test_artifact_collection +run_test 'uploaded marker atomic replacement' test_mark_artifacts_uploaded +run_test 'cleanup retention and active-job safety' test_cleanup_retention +run_test 'cleanup cron installation safety' test_cleanup_cron_installation +run_test 'Slurm TERM cancellation' test_slurm_signal_cancellation +run_test 'Slurm launch-window HUP cancellation' test_slurm_launch_window_cancellation + +echo "ALL TESTS PASSED ($tests_run)" diff --git a/ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example b/ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example new file mode 100644 index 00000000000..a3ed02839d0 --- /dev/null +++ b/ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# NVIDIA MP 0.9 archives with SAI's NCCL 2.29.3 rail-by-channel build. +export SAI_PROFILE_NAME=archive-mp09-sai-nccl2293 +export SAI_PROJECT_ROOT=${SAI_PROJECT_ROOT:-$HOME/agent/abacus_sai_gpu_ci} +export SAI_NVHPC_ROOT=/opt/devtools/nvidia/hpc_sdk/Linux_x86_64/26.3 +export SAI_CUDA_ROOT=/opt/devtools/nvidia/cuda-12.9.1 +export SAI_REQUIRE_NCCL_SAI_RAIL_BY_CHANNEL=1 +export NCCL_SAI_RAIL_BY_CHANNEL=1 +export SAI_CUSOLVERMP_ROOT=${SAI_CUSOLVERMP_ROOT:-$SAI_PROJECT_ROOT/vendor/nvidia-mp-0.9-archive/libcusolvermp-linux-x86_64-0.9.0.6427_cuda12-archive} +export SAI_CUBLASMP_ROOT=${SAI_CUBLASMP_ROOT:-$SAI_PROJECT_ROOT/vendor/nvidia-mp-0.9-archive/libcublasmp-linux-x86_64-0.9.1.3056_cuda12-archive} +export SAI_BUILD_JOBS=${SAI_BUILD_JOBS:-32} + +sai_load_toolchain() { + if [[ -n ${LD_PRELOAD:-} ]]; then + printf 'LD_PRELOAD must be empty for the SAI NCCL control: %q\n' "$LD_PRELOAD" >&2 + return 1 + fi + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} + export LD_PRELOAD= + export CPATH=${CPATH:-} + export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-} + source /etc/profile.d/lmod.sh + module purge + module load cmake/3.31.6 + module load nvhpc/26.3-gnu-cuda12-tuned + module load openmpi/5.0.10-nvhpc26.3-gnu-cuda12-auto + module load fftw/3.3.10 libxc/7.0.0-auto saiblas/2603-gnu-auto + module load elpa/2026.02.001-2603-gnu cuda/12.9.1 + + # This SAI aggregate module exports the same NCCL_ROOT as + # nccl/2.29.3-sai-cuda12.9. The standalone nccl module conflicts with it + # because the tuned module already supplies that SAI NCCL build. + : "${NCCL_ROOT:?nvhpc/26.3-gnu-cuda12-tuned must export NCCL_ROOT}" + export SAI_NCCL_ROOT=$NCCL_ROOT + SAI_MPI_ROOT=$(cd "$(dirname "$(command -v mpicc)")/.." && pwd) + export SAI_MPI_ROOT + export NCCL_ROOT=$SAI_NCCL_ROOT + export NCCL_HOME=$SAI_NCCL_ROOT + export NCCL_PATH=$SAI_NCCL_ROOT + export PATH="$SAI_MPI_ROOT/bin:$SAI_CUDA_ROOT/bin:$PATH" + export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" +} diff --git a/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/INPUT b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/INPUT new file mode 100644 index 00000000000..34984ea35c2 --- /dev/null +++ b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/INPUT @@ -0,0 +1,37 @@ +INPUT_PARAMETERS + +suffix si48_cusolvermp +pseudo_dir ../../PP_ORB +orbital_dir ../../PP_ORB + +calculation md +esolver_type tddft +md_type nve +md_nstep 1 +estep_per_md 2 +td_dt 0.005 +md_tfirst 0 + +td_vext 1 +td_vext_dire 3 +td_stype 2 +td_ttype 3 +td_tstart 1 +td_tend 2 +td_heavi_t0 1 +td_heavi_amp 0.05 +out_current 1 + +basis_type lcao +gamma_only 0 +ecutwfc 20 +scf_nmax 50 +scf_thr 1.0e-6 +device gpu +ks_solver cusolvermp + +mixing_type broyden +mixing_beta 0.3 +mixing_gg0 0.0 +cal_force 1 +cal_stress 0 diff --git a/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/KPT b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/KPT new file mode 100644 index 00000000000..c289c0158aa --- /dev/null +++ b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/KPT @@ -0,0 +1,4 @@ +K_POINTS +0 +Gamma +1 1 1 0 0 0 diff --git a/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/README b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/README new file mode 100644 index 00000000000..c72b12fe649 --- /dev/null +++ b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/README @@ -0,0 +1,8 @@ +This Si48 solid smoke case is reconstructed from the repository's Si64 PEXSI +supercell and Si RT-TDDFT GPU input. It matches the 48-atom system size used to +validate the multi-GPU RT-TDDFT implementation in PR #7026, but it is not a +copy of that unpublished benchmark input. + +The case runs only two electronic propagation steps. It checks that the +multi-node cuSolverMp RT-TDDFT path completes; it is not a physical or +performance reference calculation. diff --git a/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/STRU b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/STRU new file mode 100644 index 00000000000..704123b22e5 --- /dev/null +++ b/tests/15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU/STRU @@ -0,0 +1,68 @@ +ATOMIC_SPECIES +Si 28.085 Si_ONCV_PBE-1.0.upf + +NUMERICAL_ORBITAL +Si_gga_6au_100Ry_2s2p1d.orb + +LATTICE_CONSTANT +10.2 + +LATTICE_VECTORS +1.5 0.0 0.0 +0.0 2.0 0.0 +0.0 0.0 2.0 + +ATOMIC_POSITIONS +Cartesian + +Si +0.0 +48 +0.00 0.00 0.00 0 0 0 +0.25 0.25 0.25 0 0 0 +0.00 0.50 0.50 0 0 0 +0.25 0.75 0.75 0 0 0 +0.50 0.00 0.50 0 0 0 +0.75 0.25 0.75 0 0 0 +0.50 0.50 0.00 0 0 0 +0.75 0.75 0.25 0 0 0 +1.00 0.00 0.00 0 0 0 +1.25 0.25 0.25 0 0 0 +1.00 0.50 0.50 0 0 0 +1.25 0.75 0.75 0 0 0 +0.00 1.00 0.00 0 0 0 +0.25 1.25 0.25 0 0 0 +0.00 1.50 0.50 0 0 0 +0.25 1.75 0.75 0 0 0 +0.50 1.00 0.50 0 0 0 +0.75 1.25 0.75 0 0 0 +0.50 1.50 0.00 0 0 0 +0.75 1.75 0.25 0 0 0 +1.00 1.00 0.00 0 0 0 +1.25 1.25 0.25 0 0 0 +1.00 1.50 0.50 0 0 0 +1.25 1.75 0.75 0 0 0 +0.00 0.00 1.00 0 0 0 +0.25 0.25 1.25 0 0 0 +0.00 0.50 1.50 0 0 0 +0.25 0.75 1.75 0 0 0 +0.50 0.00 1.50 0 0 0 +0.75 0.25 1.75 0 0 0 +0.50 0.50 1.00 0 0 0 +0.75 0.75 1.25 0 0 0 +1.00 0.00 1.00 0 0 0 +1.25 0.25 1.25 0 0 0 +1.00 0.50 1.50 0 0 0 +1.25 0.75 1.75 0 0 0 +0.00 1.00 1.00 0 0 0 +0.25 1.25 1.25 0 0 0 +0.00 1.50 1.50 0 0 0 +0.25 1.75 1.75 0 0 0 +0.50 1.00 1.50 0 0 0 +0.75 1.25 1.75 0 0 0 +0.50 1.50 1.00 0 0 0 +0.75 1.75 1.25 0 0 0 +1.00 1.00 1.00 0 0 0 +1.25 1.25 1.25 0 0 0 +1.00 1.50 1.50 0 0 0 +1.25 1.75 1.75 0 0 0 From 8e479334900932028e7092fe0fce32d5f69e4cc6 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <13862180016@163.com> Date: Wed, 22 Jul 2026 18:18:23 +0800 Subject: [PATCH 2/2] ci: harden SAI transport and add local launcher --- .github/workflows/sai-bootstrap.yml | 3 +- .github/workflows/sai-gpu-full.yml | 29 +-- ci/sai/README.md | 51 +++++ ci/sai/build_gpu.sh | 2 +- ci/sai/build_source_payload.sh | 47 +++++ ci/sai/local-run.env.example | 9 + ci/sai/prepare_control_snapshot.sh | 24 +++ ci/sai/probe_remote_sai.sh | 15 +- ci/sai/run_local_ci.sh | 198 ++++++++++++++++++ ci/sai/test_gpu.sbatch | 2 +- ci/sai/test_gpu_case.sh | 11 +- ci/sai/tests/test_sai_ci.sh | 197 ++++++++++++++++- .../archive-mp09-sai-nccl2293.env.example | 2 +- 13 files changed, 555 insertions(+), 35 deletions(-) create mode 100755 ci/sai/build_source_payload.sh create mode 100644 ci/sai/local-run.env.example create mode 100755 ci/sai/prepare_control_snapshot.sh create mode 100755 ci/sai/run_local_ci.sh diff --git a/.github/workflows/sai-bootstrap.yml b/.github/workflows/sai-bootstrap.yml index dde41304fcf..2acc5525318 100644 --- a/.github/workflows/sai-bootstrap.yml +++ b/.github/workflows/sai-bootstrap.yml @@ -62,7 +62,8 @@ jobs: fi done [[ $connected -eq 1 ]] - ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s < ci/sai/probe_remote_sai.sh + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- "$SAI_SSH_USER" \ + < ci/sai/probe_remote_sai.sh - name: Install cleanup cron if: inputs.mode == 'install-cleanup' diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml index 5c5c11ecc63..85e984ccdd4 100644 --- a/.github/workflows/sai-gpu-full.yml +++ b/.github/workflows/sai-gpu-full.yml @@ -130,7 +130,7 @@ jobs: fi done [[ $connected -eq 1 ]] - ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s \ + ssh -F "$SAI_SSH_CONFIG" sai-ci bash -s -- "$SAI_SSH_USER" \ < control/ci/sai/probe_remote_sai.sh - name: Create isolated remote run @@ -184,31 +184,18 @@ jobs: set -euo pipefail payload="$RUNNER_TEMP/source-payload.gz" manifest="$RUNNER_TEMP/source-manifest.gz" - git -C source ls-tree -r -z --full-tree "$SOURCE_SHA" \ - | gzip -1 > "$manifest" - mode=full - if [[ "$SOURCE_CACHE_BASE_SHA" != none ]]; then - if git -C source cat-file -e "$SOURCE_CACHE_BASE_SHA^{commit}" 2>/dev/null || \ - git -C source fetch --no-tags --depth=1 origin "$SOURCE_CACHE_BASE_SHA"; then - git -C source diff --binary --full-index --no-renames \ - "$SOURCE_CACHE_BASE_SHA" "$SOURCE_SHA" | gzip -1 > "$payload" - mode=delta - else - echo "Cached SHA is unavailable from GitHub; sending a full snapshot" - fi - fi - if [[ "$mode" == full ]]; then - empty_tree=$(git -C source hash-object -t tree /dev/null) - git -C source diff --binary --full-index --no-renames \ - "$empty_tree" "$SOURCE_SHA" | gzip -1 > "$payload" - fi - [[ -s "$payload" && -s "$manifest" ]] + output=$(bash control/ci/sai/build_source_payload.sh \ + "$GITHUB_WORKSPACE/source" "$SOURCE_SHA" "$SOURCE_CACHE_BASE_SHA" \ + "$payload" "$manifest") + printf '%s\n' "$output" + mode=$(awk -F= '$1 == "SOURCE_PAYLOAD_MODE" {print $2}' <<< "$output") + [[ "$mode" == full || "$mode" == delta ]] { echo "SOURCE_PAYLOAD=$payload" echo "SOURCE_MANIFEST=$manifest" echo "SOURCE_PAYLOAD_MODE=$mode" } >> "$GITHUB_ENV" - echo "SOURCE_PAYLOAD_READY mode=$mode base=$SOURCE_CACHE_BASE_SHA target=$SOURCE_SHA bytes=$(stat -c %s "$payload")" + echo "SOURCE_PAYLOAD_READY mode=$mode base=$SOURCE_CACHE_BASE_SHA target=$SOURCE_SHA" - name: Upload and apply compressed source payload run: | diff --git a/ci/sai/README.md b/ci/sai/README.md index 3efb86b2d22..e02ddb33366 100644 --- a/ci/sai/README.md +++ b/ci/sai/README.md @@ -87,6 +87,57 @@ rejects any path that escapes the account's canonical HOME. Only 6. Submit the workflow. A required reviewer then opens the pending deployment, checks the requested SHA and directory, and approves `sai-ssh-manual`. +### Direct local run + +Developers with their own SAI account may launch the same remote build and +Slurm validation directly, without GitHub Actions. The local client uses an +existing OpenSSH configuration and never reads or copies the private key +itself. For example: + +```sshconfig +Host SAI-abacus + HostName c0.sai.ai-4s.com + Port 12022 + User your-sai-user + IdentityFile ~/.ssh/your-sai-key + IdentitiesOnly yes + StrictHostKeyChecking yes +``` + +Copy `ci/sai/local-run.env.example` outside the ABACUS checkout and set +`SAI_SSH_CONFIG` to the OpenSSH configuration file, `SAI_SSH_TARGET` to its +Host alias, and `SAI_PROJECT_ROOT` to an absolute directory below that remote +account's HOME. Do not put the local configuration or private key in the +repository. Validate access without creating a run: + +```bash +bash ci/sai/run_local_ci.sh /path/to/local-run.env --probe-only +``` + +Run the full validation with: + +```bash +bash ci/sai/run_local_ci.sh /path/to/local-run.env +``` + +The control checkout must have no tracked changes or non-ignored untracked +files. The client materializes `ci/sai` from the recorded `CONTROL_SHA` with +`git archive`, so ignored files and a check-to-upload working-tree race cannot +alter or add remote control files. `SAI_SOURCE_SHA` defaults to `HEAD` and may +name any commit already available in the local Git object database. The client +uploads the same gzip-compressed Git delta and manifest used by the GitHub +workflow, executes the same remote control scripts, and downloads the artifact +bundle below `SAI_ARTIFACT_ROOT//`. A local run always has the +`candidate` cache role: it may consume an existing daily baseline but never +advances it. + +The OpenSSH Host entry determines the remote username and identity file. The +client additionally forces batch mode, strict host-key checking, disabled +agent/port forwarding, and a temporary multiplexed control socket. The local +developer is responsible for reviewing `SAI_SOURCE_SHA`, because that commit's +build system, integration scripts, and binaries execute with the permissions +of the configured SAI account. + The selected directory is a reusable project root, not a checkout directory. Each attempt uses a new `runs//-` subdirectory. Official NVIDIA diff --git a/ci/sai/build_gpu.sh b/ci/sai/build_gpu.sh index 4ff77af5372..8b22e7a98ca 100755 --- a/ci/sai/build_gpu.sh +++ b/ci/sai/build_gpu.sh @@ -77,7 +77,7 @@ export NVHPC_ROOT_DIR="$SAI_NVHPC_ROOT" export CUSOLVERMP_PATH="$SAI_CUSOLVERMP_ROOT" export CUBLASMP_PATH="$SAI_CUBLASMP_ROOT" export NCCL_PATH="$SAI_NCCL_ROOT" -export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" cat > "$BUILD_ROOT/runtime-version-probe.c" <<'EOF' #include diff --git a/ci/sai/build_source_payload.sh b/ci/sai/build_source_payload.sh new file mode 100755 index 00000000000..376c08d9daa --- /dev/null +++ b/ci/sai/build_source_payload.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 5 ]]; then + echo "Usage: $0 REPOSITORY SOURCE_SHA BASE_SHA|none PAYLOAD MANIFEST" >&2 + exit 2 +fi + +repository=$(realpath -e "$1") +source_sha=$2 +base_sha=$3 +payload=$4 +manifest=$5 + +[[ -d $repository/.git || -f $repository/.git ]] +[[ $source_sha =~ ^[0-9a-f]{40}$ ]] +[[ $base_sha == none || $base_sha =~ ^[0-9a-f]{40}$ ]] +[[ $payload != "$manifest" ]] +git -C "$repository" cat-file -e "$source_sha^{commit}" + +mkdir -p "$(dirname "$payload")" "$(dirname "$manifest")" +git -C "$repository" ls-tree -r -z --full-tree "$source_sha" \ + | gzip -1 > "$manifest" + +mode="full" +if [[ $base_sha != none ]]; then + if git -C "$repository" cat-file -e "$base_sha^{commit}" 2>/dev/null || \ + git -C "$repository" fetch --no-tags --depth=1 origin "$base_sha"; then + git -C "$repository" diff --binary --full-index --no-renames \ + "$base_sha" "$source_sha" | gzip -1 > "$payload" + mode="delta" + else + echo "Cached SHA is unavailable; sending a full snapshot" >&2 + fi +fi +if [[ $mode == full ]]; then + empty_tree=$(git -C "$repository" hash-object -t tree /dev/null) + git -C "$repository" diff --binary --full-index --no-renames \ + "$empty_tree" "$source_sha" | gzip -1 > "$payload" +fi + +[[ -s $payload && -s $manifest ]] +gzip -t -- "$payload" +gzip -t -- "$manifest" +printf 'SOURCE_PAYLOAD_MODE=%s\n' "$mode" +printf 'SOURCE_PAYLOAD_BYTES=%s\n' "$(stat -c %s "$payload")" diff --git a/ci/sai/local-run.env.example b/ci/sai/local-run.env.example new file mode 100644 index 00000000000..7f7538bf96b --- /dev/null +++ b/ci/sai/local-run.env.example @@ -0,0 +1,9 @@ +# Copy this file outside the ABACUS checkout before editing it. +# The private key stays in the OpenSSH configuration referenced below. +SAI_SSH_CONFIG=$HOME/.ssh/config +SAI_SSH_TARGET=SAI-abacus +SAI_PROJECT_ROOT=/home/your-sai-user/agent/abacus_sai_gpu_ci +SAI_RUN_NAMESPACE=local +SAI_ARTIFACT_ROOT=$HOME/abacus-sai-artifacts +# SAI_SOURCE_SHA defaults to HEAD and may be set to any locally available commit. +# SAI_SOURCE_SHA=0123456789abcdef0123456789abcdef01234567 diff --git a/ci/sai/prepare_control_snapshot.sh b/ci/sai/prepare_control_snapshot.sh new file mode 100755 index 00000000000..ed024ffac16 --- /dev/null +++ b/ci/sai/prepare_control_snapshot.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 REPOSITORY CONTROL_SHA OUTPUT_DIRECTORY" >&2 + exit 2 +fi + +repository=$(realpath -e "$1") +control_sha=$2 +output_dir=$(realpath --canonicalize-missing "$3") +[[ -d $repository/.git || -f $repository/.git ]] +[[ $control_sha =~ ^[0-9a-f]{40}$ ]] +[[ ! -e $output_dir && ! -L $output_dir ]] +git -C "$repository" cat-file -e "$control_sha^{commit}" + +mkdir -p "$output_dir" +git -C "$repository" archive --format=tar "$control_sha" -- ci/sai \ + | tar -xf - -C "$output_dir" +control_root=$(realpath -e "$output_dir/ci/sai") +[[ $control_root == "$output_dir/ci/sai" ]] +[[ -f $control_root/run_remote_ci.sh && ! -L $control_root/run_remote_ci.sh ]] +printf 'CONTROL_ROOT=%s\n' "$control_root" diff --git a/ci/sai/probe_remote_sai.sh b/ci/sai/probe_remote_sai.sh index 84657163b3d..cfc02ce5cea 100755 --- a/ci/sai/probe_remote_sai.sh +++ b/ci/sai/probe_remote_sai.sh @@ -2,13 +2,20 @@ set -euo pipefail -[[ $USER == abacususer01 ]] -[[ $HOME == /home/abacus-group/abacususer01 ]] -[[ $(id -u) -eq 1478400356 ]] +if [[ $# -ne 1 ]]; then + echo "Usage: $0 EXPECTED_USER" >&2 + exit 2 +fi +expected_user=$1 +[[ $expected_user =~ ^[A-Za-z0-9._-]+$ ]] +remote_user=$(id -un) +[[ $remote_user == "$expected_user" ]] +canonical_home=$(cd "$HOME" && pwd -P) +[[ $canonical_home == "$HOME" ]] for command_name in sbatch sacct squeue scancel rsync curl git gzip tar xz \ realpath flock crontab; do command -v "$command_name" >/dev/null done sinfo -h -p 16V100 -o '%P %a %D %G' | grep -q '^16V100 ' -echo "SAI_SSH_PROBE_OK user=$USER home=$HOME host=$(hostname)" +echo "SAI_SSH_PROBE_OK user=$remote_user home=$canonical_home host=$(hostname)" sinfo -h -p 16V100 -o 'SAI_PARTITION=%P state=%a nodes=%D gres=%G' diff --git a/ci/sai/run_local_ci.sh b/ci/sai/run_local_ci.sh new file mode 100755 index 00000000000..614ef10b801 --- /dev/null +++ b/ci/sai/run_local_ci.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -lt 1 || $# -gt 2 || ( $# -eq 2 && $2 != --probe-only ) ]]; then + echo "Usage: $0 CONFIG_FILE [--probe-only]" >&2 + exit 2 +fi + +config_file=$(realpath -e "$1") +[[ -f $config_file ]] +# shellcheck source=/dev/null +source "$config_file" + +: "${SAI_SSH_CONFIG:?Set SAI_SSH_CONFIG in the local configuration}" +: "${SAI_SSH_TARGET:?Set SAI_SSH_TARGET in the local configuration}" +[[ $SAI_SSH_TARGET =~ ^[A-Za-z0-9._-]+$ ]] +ssh_config=$(realpath -e "$SAI_SSH_CONFIG") +[[ -f $ssh_config ]] +[[ $ssh_config =~ ^/[A-Za-z0-9._/-]+$ ]] || { + echo "SAI_SSH_CONFIG contains unsupported characters: $ssh_config" >&2 + exit 2 +} + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +repository=$(cd "$script_dir/../.." && pwd -P) +for command_name in ssh rsync git gzip tar realpath awk; do + command -v "$command_name" >/dev/null +done + +client_root=$(mktemp -d) +control_path="$client_root/control-%C" +ssh_options=( + -F "$ssh_config" + -o BatchMode=yes + -o StrictHostKeyChecking=yes + -o ForwardAgent=no + -o ClearAllForwardings=yes + -o RequestTTY=no + -o Compression=yes + -o ControlMaster=auto + -o ControlPath="$control_path" + -o ControlPersist=15m +) +rsync_rsh="ssh -F $ssh_config -o BatchMode=yes -o StrictHostKeyChecking=yes -o ForwardAgent=no -o ClearAllForwardings=yes -o RequestTTY=no -o Compression=yes -o ControlMaster=auto -o ControlPath=$control_path -o ControlPersist=15m" + +cleanup() { + ssh "${ssh_options[@]}" -O exit "$SAI_SSH_TARGET" >/dev/null 2>&1 || true + rm -rf "$client_root" +} +trap cleanup EXIT + +if [[ ${2:-} != --probe-only ]] && + [[ -n $(git -C "$repository" status --porcelain --untracked-files=all) ]]; then + echo "The local control checkout must be clean, including untracked files" >&2 + exit 1 +fi +control_sha=$(git -C "$repository" rev-parse --verify "HEAD^{commit}") +[[ $control_sha =~ ^[0-9a-f]{40}$ ]] +snapshot_output=$(bash "$script_dir/prepare_control_snapshot.sh" \ + "$repository" "$control_sha" "$client_root/control-snapshot") +control_root=$(awk -F= '$1 == "CONTROL_ROOT" {print $2}' <<< "$snapshot_output") +[[ $control_root == "$client_root/control-snapshot/ci/sai" ]] + +ssh_expanded=$(ssh -G "${ssh_options[@]}" "$SAI_SSH_TARGET") +remote_user=$(awk '$1 == "user" {print $2; exit}' <<< "$ssh_expanded") +[[ $remote_user =~ ^[A-Za-z0-9._-]+$ ]] + +connected=0 +for attempt in {1..4}; do + if ssh "${ssh_options[@]}" -o ConnectionAttempts=1 -MNf \ + "$SAI_SSH_TARGET"; then + connected=1 + break + fi + echo "SAI SSH master connection failed (attempt $attempt/4)" >&2 + if [[ $attempt -lt 4 ]]; then + sleep $((attempt * 5)) + fi +done +[[ $connected -eq 1 ]] +ssh "${ssh_options[@]}" "$SAI_SSH_TARGET" bash -s -- "$remote_user" \ + < "$control_root/probe_remote_sai.sh" + +if [[ ${2:-} == --probe-only ]]; then + echo "SAI_LOCAL_PROBE_OK target=$SAI_SSH_TARGET user=$remote_user" + exit 0 +fi + +: "${SAI_PROJECT_ROOT:?Set SAI_PROJECT_ROOT in the local configuration}" +SAI_RUN_NAMESPACE=${SAI_RUN_NAMESPACE:-local} +SAI_ARTIFACT_ROOT=${SAI_ARTIFACT_ROOT:-$PWD/sai-local-artifacts} +SAI_SOURCE_SHA=${SAI_SOURCE_SHA:-HEAD} + +[[ $SAI_PROJECT_ROOT =~ ^(/[A-Za-z0-9._-]+)+$ ]] +case "/$SAI_PROJECT_ROOT/" in + */./*|*/../*) + echo "SAI_PROJECT_ROOT contains a dot path component" >&2 + exit 2 + ;; +esac +[[ $SAI_RUN_NAMESPACE =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] + +source_sha=$(git -C "$repository" rev-parse --verify "$SAI_SOURCE_SHA^{commit}") +[[ $source_sha =~ ^[0-9a-f]{40}$ ]] + +run_id=$(date +%s) +run_attempt=$$ +run_key="$run_id-$run_attempt" +artifact_parent=$SAI_ARTIFACT_ROOT +if [[ $artifact_parent != /* ]]; then + artifact_parent=$PWD/$artifact_parent +fi +artifact_parent=$(realpath --canonicalize-missing "$artifact_parent") +artifact_root="$artifact_parent/$run_key" +[[ ! -e $artifact_root && ! -L $artifact_root ]] +mkdir -p "$artifact_root" +artifact_root=$(cd "$artifact_root" && pwd -P) + +run_remote_script() { + local script=$1 + shift + ssh "${ssh_options[@]}" "$SAI_SSH_TARGET" bash -s -- "$@" < "$script" +} + +output=$(run_remote_script "$control_root/prepare_remote_run.sh" \ + "$SAI_PROJECT_ROOT" "$SAI_RUN_NAMESPACE" "$run_key" \ + "$source_sha" "$control_sha") +printf '%s\n' "$output" +remote_project_root=$(awk -F= '$1 == "SAI_PROJECT_ROOT" {print $2}' <<< "$output") +remote_run_root=$(awk -F= '$1 == "RUN_ROOT" {print $2}' <<< "$output") +[[ $remote_project_root == "$SAI_PROJECT_ROOT" ]] +[[ $remote_run_root == "$remote_project_root/runs/$SAI_RUN_NAMESPACE/$run_key" ]] + +RSYNC_RSH=$rsync_rsh rsync -az --delete --timeout=600 --stats \ + "$control_root/" "$SAI_SSH_TARGET:$remote_run_root/control/" + +output=$(run_remote_script "$control_root/source_transfer_cache.sh" \ + prepare "$remote_project_root" "$remote_run_root" "$source_sha" candidate) +printf '%s\n' "$output" +transfer_root=$(awk -F= '$1 == "SOURCE_TRANSFER_ROOT" {print $2}' <<< "$output") +base_sha=$(awk -F= '$1 == "SOURCE_CACHE_BASE_SHA" {print $2}' <<< "$output") +[[ $transfer_root == "$remote_project_root/cache/source-transfers/$run_key" ]] +[[ $base_sha == none || $base_sha =~ ^[0-9a-f]{40}$ ]] + +payload="$client_root/source-payload.gz" +manifest="$client_root/source-manifest.gz" +payload_output=$(bash "$control_root/build_source_payload.sh" \ + "$repository" "$source_sha" "$base_sha" "$payload" "$manifest") +printf '%s\n' "$payload_output" +payload_mode=$(awk -F= '$1 == "SOURCE_PAYLOAD_MODE" {print $2}' \ + <<< "$payload_output") +[[ $payload_mode == full || $payload_mode == delta ]] + +RSYNC_RSH=$rsync_rsh rsync -a --partial --timeout=600 --stats \ + "$payload" "$manifest" "$SAI_SSH_TARGET:$transfer_root/" +run_remote_script "$control_root/source_transfer_cache.sh" \ + receive "$remote_project_root" "$remote_run_root" "$transfer_root" \ + "$payload_mode" "$source_sha" +run_remote_script "$control_root/source_transfer_cache.sh" \ + finalize "$remote_project_root" "$remote_run_root" "$transfer_root" \ + "$source_sha" + +{ + printf 'source_sha=%s\n' "$source_sha" + printf 'control_sha=%s\n' "$control_sha" + printf 'remote_user=%s\n' "$remote_user" + printf 'remote_run_root=%s\n' "$remote_run_root" +} > "$artifact_root/local-run-context.txt" + +set +e +run_remote_script "$control_root/run_remote_ci.sh" \ + "$remote_project_root" "$remote_run_root" "$source_sha" "$control_sha" \ + "$run_id" "$run_attempt" 2>&1 | tee "$artifact_root/sai-remote-driver.log" +validation_rc=${PIPESTATUS[0]} +set -e + +set +e +run_remote_script "$control_root/collect_remote_artifacts.sh" "$remote_run_root" \ + | tar -xzf - -C "$artifact_root" +collection_status=("${PIPESTATUS[@]}") +set -e +collection_rc=0 +for status in "${collection_status[@]}"; do + if [[ $status -ne 0 ]]; then + collection_rc=$status + break + fi +done +if [[ $collection_rc -eq 0 ]]; then + run_remote_script "$control_root/mark_artifacts_uploaded.sh" "$remote_run_root" +fi + +echo "SAI_LOCAL_RUN_RESULT validation_rc=$validation_rc collection_rc=$collection_rc" +echo "SAI_LOCAL_ARTIFACT_ROOT=$artifact_root" +echo "SAI_REMOTE_RUN_ROOT=$remote_run_root" +[[ $collection_rc -eq 0 ]] +exit "$validation_rc" diff --git a/ci/sai/test_gpu.sbatch b/ci/sai/test_gpu.sbatch index 4606bc6f62e..212c3905369 100755 --- a/ci/sai/test_gpu.sbatch +++ b/ci/sai/test_gpu.sbatch @@ -43,7 +43,7 @@ launcher_dir="$RESULT_ROOT/launcher" mkdir -p "$launcher_dir" ln -sfn "$CONTROL_ROOT/mpirun_with_mapping.sh" "$launcher_dir/mpirun" export PATH="$launcher_dir:$PATH" -export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" echo "SAI runtime mapping: partition=$SLURM_JOB_PARTITION tasks=$SLURM_NTASKS map_by=$MAP_OPT omp_threads=$OMP_NUM_THREADS" scontrol show hostnames "$SLURM_JOB_NODELIST" | sed 's/^/SLURM_NODE=/' diff --git a/ci/sai/test_gpu_case.sh b/ci/sai/test_gpu_case.sh index 91c8d9a08d4..4194b12dc2f 100755 --- a/ci/sai/test_gpu_case.sh +++ b/ci/sai/test_gpu_case.sh @@ -40,9 +40,16 @@ mkdir -p "$work_root/tests/$suite" "$status_root" "$task_root/launcher" start_epoch=$(date +%s) echo "SAI_GPU_CASE_START class=$GPU_CASE_CLASS task=$SLURM_ARRAY_TASK_ID suite=$suite case=$case_name ranks=$GPU_CASE_RANKS" +source_tests=$(realpath -e "$CI_SOURCE/tests") +source_case=$(realpath -e "$CI_SOURCE/tests/$suite/$case_name") +[[ $source_case == "$source_tests/$suite/$case_name" ]] +if find "$source_case" -type l -print -quit | grep -q .; then + echo "GPU case contains a symbolic link: $suite/$case_name" >&2 + exit 2 +fi ln -s "$CI_SOURCE/tests/integrate" "$work_root/tests/integrate" ln -s "$CI_SOURCE/tests/PP_ORB" "$work_root/tests/PP_ORB" -rsync -a "$CI_SOURCE/tests/$suite/$case_name" "$work_root/tests/$suite/" +rsync -a "$source_case" "$work_root/tests/$suite/" printf '%s\n' "$case_name" > "$work_root/tests/$suite/CASES.task.txt" source "$TOOLCHAIN_FILE" @@ -55,7 +62,7 @@ SAI_SYSTEM_MPIRUN=$(command -v mpirun) export SAI_SYSTEM_MPIRUN MAP_OPT ln -s "$CONTROL_ROOT/mpirun_with_mapping.sh" "$task_root/launcher/mpirun" export PATH="$task_root/launcher:$PATH" -export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ABACUS="$INSTALL_ROOT/bin/abacus" [[ -x "$ABACUS" ]] diff --git a/ci/sai/tests/test_sai_ci.sh b/ci/sai/tests/test_sai_ci.sh index 72ed1ad8631..266841a8c30 100755 --- a/ci/sai/tests/test_sai_ci.sh +++ b/ci/sai/tests/test_sai_ci.sh @@ -91,10 +91,149 @@ test_configure_ssh_client() { assert_not_contains "$log" 'sai-ci-secret-marker' } +test_build_source_payload() { + local root=$test_root/source-payload + local repository=$root/repository + local sha1 sha2 output + mkdir -p "$repository" "$root/full" "$root/delta" + git -C "$repository" init -q + git -C "$repository" config user.email ci@example.invalid + git -C "$repository" config user.name ci + printf 'first\n' > "$repository/data.txt" + git -C "$repository" add data.txt + git -C "$repository" commit -q -m first + sha1=$(git -C "$repository" rev-parse HEAD) + printf 'second\n' > "$repository/data.txt" + printf 'new\n' > "$repository/new.txt" + git -C "$repository" add data.txt new.txt + git -C "$repository" commit -q -m second + sha2=$(git -C "$repository" rev-parse HEAD) + + output=$(bash ci/sai/build_source_payload.sh "$repository" "$sha1" none \ + "$root/full/source-payload.gz" "$root/full/source-manifest.gz") + grep -Fxq 'SOURCE_PAYLOAD_MODE=full' <<< "$output" + cmp <(git -C "$repository" ls-tree -r -z --full-tree "$sha1") \ + <(gzip -cd "$root/full/source-manifest.gz") + empty_tree=$(git -C "$repository" hash-object -t tree /dev/null) + cmp <(git -C "$repository" diff --binary --full-index --no-renames \ + "$empty_tree" "$sha1") \ + <(gzip -cd "$root/full/source-payload.gz") + + output=$(bash ci/sai/build_source_payload.sh "$repository" "$sha2" "$sha1" \ + "$root/delta/source-payload.gz" "$root/delta/source-manifest.gz") + grep -Fxq 'SOURCE_PAYLOAD_MODE=delta' <<< "$output" + cmp <(git -C "$repository" diff --binary --full-index --no-renames \ + "$sha1" "$sha2") \ + <(gzip -cd "$root/delta/source-payload.gz") +} + +test_prepare_control_snapshot() { + local root=$test_root/control-snapshot + local repository=$root/repository + local output=$root/output + local control_sha result control_root + mkdir -p "$repository/ci/sai" + git -C "$repository" init -q + git -C "$repository" config user.email ci@example.invalid + git -C "$repository" config user.name ci + printf '*.log\n' > "$repository/.gitignore" + printf '#!/usr/bin/env bash\nexit 0\n' \ + > "$repository/ci/sai/run_remote_ci.sh" + chmod +x "$repository/ci/sai/run_remote_ci.sh" + printf 'tracked\n' > "$repository/ci/sai/tracked.txt" + git -C "$repository" add .gitignore ci/sai + git -C "$repository" commit -q -m control + control_sha=$(git -C "$repository" rev-parse HEAD) + printf 'ignored secret\n' > "$repository/ci/sai/private.log" + printf 'untracked\n' > "$repository/ci/sai/untracked.txt" + + result=$(bash ci/sai/prepare_control_snapshot.sh \ + "$repository" "$control_sha" "$output") + control_root=$(awk -F= '$1 == "CONTROL_ROOT" {print $2}' <<< "$result") + [[ $control_root == "$output/ci/sai" ]] + assert_file "$control_root/run_remote_ci.sh" + assert_file "$control_root/tracked.txt" + [[ -x $control_root/run_remote_ci.sh ]] + assert_not_exists "$control_root/private.log" + assert_not_exists "$control_root/untracked.txt" +} + +test_remote_probe_identity() { + local root=$test_root/remote-probe + local fake_bin=$root/bin + local command_name + mkdir -p "$fake_bin" "$root/home" + cat > "$fake_bin/id" <<'EOF' +#!/usr/bin/env bash +if [[ ${1:-} == -un ]]; then + echo localuser +else + exec /usr/bin/id "$@" +fi +EOF + cat > "$fake_bin/sinfo" <<'EOF' +#!/usr/bin/env bash +echo '16V100 up 1 gpu:8' +EOF + chmod +x "$fake_bin/id" "$fake_bin/sinfo" + for command_name in sbatch sacct squeue scancel crontab; do + cat > "$fake_bin/$command_name" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + chmod +x "$fake_bin/$command_name" + done + + PATH="$fake_bin:$original_path" HOME=$root/home \ + bash ci/sai/probe_remote_sai.sh localuser > "$root/probe.log" + assert_contains "$root/probe.log" 'SAI_SSH_PROBE_OK user=localuser' + if PATH="$fake_bin:$original_path" HOME=$root/home \ + bash ci/sai/probe_remote_sai.sh wronguser > /dev/null 2>&1; then + fail 'remote probe accepted the wrong expected user' + fi +} + +test_local_client_probe() { + local root=$test_root/local-client + local fake_bin=$root/bin + local ssh_config=$root/ssh-config + local local_config=$root/local-run.env + local ssh_log=$root/ssh.log + mkdir -p "$fake_bin" + : > "$ssh_config" + cat > "$local_config" < "$fake_bin/ssh" <> "$ssh_log" +case " \$* " in + *' -G '*) echo 'user localuser'; exit 0 ;; + *' -MNf '*) exit 0 ;; + *' -O exit '*) exit 0 ;; +esac +cat >/dev/null +echo 'SAI_SSH_PROBE_OK user=localuser' +EOF + chmod +x "$fake_bin/ssh" + + PATH="$fake_bin:$original_path" \ + bash ci/sai/run_local_ci.sh "$local_config" --probe-only \ + > "$root/local-probe.log" + assert_contains "$root/local-probe.log" \ + 'SAI_LOCAL_PROBE_OK target=test-sai user=localuser' + assert_contains "$ssh_log" 'StrictHostKeyChecking=yes' + assert_contains "$ssh_log" 'ForwardAgent=no' + assert_contains "$ssh_log" 'ClearAllForwardings=yes' +} + test_workflow_security_policy() { local workflow=.github/workflows/sai-gpu-full.yml local bootstrap=.github/workflows/sai-bootstrap.yml local toolchain=ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example + local payload_builder=ci/sai/build_source_payload.sh + local runtime_file assert_contains "$workflow" 'cron: "30 20 * * *"' assert_contains "$workflow" "name: \${{ github.event_name == 'schedule' && 'sai-ssh-scheduled' || 'sai-ssh-manual' }}" assert_contains "$workflow" "group: sai-gpu-\${{ github.event_name == 'schedule' && 'daily' || github.run_id }}" @@ -117,10 +256,11 @@ test_workflow_security_policy() { assert_not_contains "$toolchain" 'module load nccl/' assert_not_contains "$toolchain" 'export SAI_NCCL_ROOT=/opt/' assert_contains "$workflow" 'source_transfer_cache.sh' - assert_contains "$workflow" 'git -C source diff --binary --full-index --no-renames' - assert_contains "$workflow" 'empty_tree=$(git -C source hash-object -t tree /dev/null)' - assert_contains "$workflow" 'git -C source ls-tree -r -z --full-tree "$SOURCE_SHA"' - assert_contains "$workflow" '| gzip -1 > "$payload"' + assert_contains "$workflow" 'build_source_payload.sh' + assert_contains "$payload_builder" 'diff --binary --full-index --no-renames' + assert_contains "$payload_builder" 'empty_tree=$(git -C "$repository" hash-object -t tree /dev/null)' + assert_contains "$payload_builder" 'ls-tree -r -z --full-tree "$source_sha"' + assert_contains "$payload_builder" '| gzip -1 > "$payload"' assert_contains "$workflow" 'SOURCE_CACHE_BASE_SHA' assert_contains "$workflow" 'run_namespace=${RUN_NAMESPACE_INPUT:-manual}' assert_contains "$workflow" 'source_cache_role=baseline' @@ -133,6 +273,22 @@ test_workflow_security_policy() { assert_contains "$workflow" '"sai-ci:$REMOTE_SOURCE_TRANSFER_ROOT/"' assert_not_contains "$workflow" '"sai-ci:$REMOTE_RUN_ROOT/source/"' assert_contains ci/sai/probe_remote_sai.sh 'rsync curl git gzip tar xz' + assert_contains "$workflow" 'bash -s -- "$SAI_SSH_USER"' + assert_contains "$bootstrap" 'bash -s -- "$SAI_SSH_USER"' + assert_not_contains ci/sai/probe_remote_sai.sh '1478400356' + for runtime_file in ci/sai/build_gpu.sh ci/sai/test_gpu.sbatch \ + ci/sai/test_gpu_case.sh "$toolchain"; do + assert_contains "$runtime_file" '${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}' + assert_not_contains "$runtime_file" ':${LD_LIBRARY_PATH:-}' + done + assert_contains ci/sai/run_local_ci.sh 'source_transfer_cache.sh' + assert_contains ci/sai/run_local_ci.sh '"$source_sha" candidate' + assert_contains ci/sai/run_local_ci.sh 'status --porcelain --untracked-files=all' + assert_contains ci/sai/run_local_ci.sh 'prepare_control_snapshot.sh' + assert_contains ci/sai/run_local_ci.sh '"$control_root/"' + assert_contains ci/sai/run_local_ci.sh '< "$control_root/probe_remote_sai.sh"' + assert_not_contains ci/sai/run_local_ci.sh '< "$script_dir/probe_remote_sai.sh"' + assert_not_contains ci/sai/local-run.env.example 'PRIVATE KEY' if sed -n '/^on:/,/^permissions:/p' "$workflow" | grep -Eq '^[[:space:]]+pull_request:'; then fail 'GPU workflow must not run automatically for pull requests' fi @@ -141,8 +297,11 @@ test_workflow_security_policy() { test_control_executable_modes() { local path mode for path in \ + ci/sai/build_source_payload.sh \ ci/sai/build_gpu.sbatch \ ci/sai/mpirun_with_mapping.sh \ + ci/sai/prepare_control_snapshot.sh \ + ci/sai/run_local_ci.sh \ ci/sai/run_slurm_job.sh \ ci/sai/test_gpu.sbatch \ ci/sai/test_gpu_case.sh; do @@ -303,6 +462,31 @@ test_gpu_matrix_submission_policy() { assert_contains "$summary" '^(PASS|FAIL|TIMEOUT|INFRA)$' } +test_gpu_case_symlink_rejection() { + local root=$test_root/gpu-case-symlink + local source=$root/source + local case_dir=$source/tests/suite/case + local manifest=$root/manifest.tsv + local outside=$root/outside + mkdir -p "$case_dir" "$source/tests/integrate" "$source/tests/PP_ORB" \ + "$root/results" "$root/install" "$root/control" + printf 'outside\n' > "$outside" + ln -s "$outside" "$case_dir/INPUT" + printf 'suite\tcase\n' > "$manifest" + + if CI_SOURCE=$source CONTROL_ROOT=$root/control \ + INSTALL_ROOT=$root/install RESULT_ROOT=$root/results \ + TOOLCHAIN_FILE=$root/missing-toolchain MP_PROFILE=test \ + GPU_CASE_CLASS=gpu1 GPU_CASE_RANKS=1 \ + GPU_CASE_MANIFEST=$manifest SLURM_ARRAY_JOB_ID=1 \ + SLURM_ARRAY_TASK_ID=0 SLURM_NTASKS=1 SLURM_GPUS_ON_NODE=1 \ + bash ci/sai/test_gpu_case.sh > "$root/case.log" 2>&1; then + fail 'GPU case launcher accepted a symbolic link' + fi + assert_contains "$root/case.log" 'GPU case contains a symbolic link' + [[ $(<"$outside") == outside ]] +} + test_prepare_cusolvermp_smoke() { local root=$test_root/cusolvermp-smoke local source=$root/source @@ -1243,10 +1427,15 @@ EOF } run_test 'SSH client configuration' test_configure_ssh_client +run_test 'source payload builder' test_build_source_payload +run_test 'committed control snapshot' test_prepare_control_snapshot +run_test 'remote probe identity' test_remote_probe_identity +run_test 'local client SSH probe' test_local_client_probe run_test 'workflow security policy' test_workflow_security_policy run_test 'control executable modes' test_control_executable_modes run_test 'PMIx startup retry policy' test_pmix_startup_retry run_test 'GPU matrix submission policy' test_gpu_matrix_submission_policy +run_test 'GPU case symlink rejection' test_gpu_case_symlink_rejection run_test 'cuSolverMp smoke staging' test_prepare_cusolvermp_smoke run_test 'remote path containment and collision' test_prepare_remote_run_paths run_test 'compressed source snapshot cache' test_source_snapshot_cache diff --git a/ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example b/ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example index a3ed02839d0..d14adfcc988 100644 --- a/ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example +++ b/ci/sai/toolchains/archive-mp09-sai-nccl2293.env.example @@ -39,5 +39,5 @@ sai_load_toolchain() { export NCCL_HOME=$SAI_NCCL_ROOT export NCCL_PATH=$SAI_NCCL_ROOT export PATH="$SAI_MPI_ROOT/bin:$SAI_CUDA_ROOT/bin:$PATH" - export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64:${LD_LIBRARY_PATH:-}" + export LD_LIBRARY_PATH="$SAI_MPI_ROOT/lib:$SAI_CUDA_ROOT/lib64:$SAI_CUSOLVERMP_ROOT/lib:$SAI_CUBLASMP_ROOT/lib:$SAI_NCCL_ROOT/lib:$SAI_NVHPC_ROOT/math_libs/12.9/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" }