This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Child Pipeline | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| matrix_json: | ||
| description: "Matrix JSON generated from .ci/config.yaml" | ||
| required: true | ||
| type: string | ||
| jobs: | ||
| platform-jobs: | ||
| # Matrix expands into independent jobs (one per platform/job entry), | ||
| # not serial steps in a single job. If a platform runner is busy/offline, | ||
| # only that matrix entry stays queued. | ||
| name: ${{ matrix.id }} | ||
| strategy: | ||
| fail-fast: false | ||
| max-parallel: 5 | ||
| matrix: ${{ fromJson(inputs.matrix_json) }} | ||
| runs-on: [self-hosted, linux, ${{ matrix.platform }}] | ||
| timeout-minutes: ${{ matrix.timeout_minutes }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| submodules: recursive | ||
| - name: Show stage plan | ||
| env: | ||
| MATRIX_JSON: ${{ inputs.matrix_json }} | ||
| CURRENT_JOB: ${{ matrix.id }} | ||
| run: | | ||
| python3 - <<'PY' | ||
| import json | ||
| import os | ||
| data = json.loads(os.environ["MATRIX_JSON"]) | ||
| current = os.environ["CURRENT_JOB"] | ||
| entry = next((x for x in data.get("include", []) if x.get("id") == current), None) | ||
| if not entry: | ||
| print(f"[warn] matrix entry not found for {current}") | ||
| raise SystemExit(0) | ||
| test_cmd = entry.get("test_cmd", "") | ||
| lines = [ | ||
| f"job: {entry.get('id')}", | ||
| f"platform: {entry.get('platform')}", | ||
| "stage order:", | ||
| " 1) setup", | ||
| f" 2) test -> {test_cmd}", | ||
| ] | ||
| text = "\n".join(lines) | ||
| print(text) | ||
| summary = os.environ.get("GITHUB_STEP_SUMMARY") | ||
| if summary: | ||
| with open(summary, "a", encoding="utf-8") as f: | ||
| f.write("### Stage plan\n\n") | ||
| f.write("```\n") | ||
| f.write(text) | ||
| f.write("\n```\n\n") | ||
| PY | ||
| - name: Build image via .ci/build.py | ||
| run: | | ||
| python3 .ci/build.py \ | ||
| --config .ci/config.yaml \ | ||
| --platform "${{ matrix.platform }}" \ | ||
| --force | ||
| - name: Build docker args (same helpers as .ci/run.py) and write GITHUB_ENV | ||
| env: | ||
| CONFIG: .ci/config.yaml | ||
| BRANCH: None | ||
| STAGE: None | ||
| IMAGE_TAG: None | ||
| GPU_ID: all | ||
| RESULT_DIR: None | ||
| TEST: None | ||
| LOCAL: True | ||
| run: | | ||
| PYTHONPATH=.ci python3 - <<'PY' | ||
| import os | ||
| import shlex | ||
| import sys | ||
| import uuid | ||
| from pathlib import Path | ||
| from run import ( | ||
| apply_test_override, | ||
| build_docker_args, | ||
| build_results_dir, | ||
| resolve_image, | ||
| ) | ||
| from utils import get_git_commit, load_config | ||
| def env_opt(key): | ||
| v = os.environ.get(key, "") | ||
| return None if not v or v == "None" else v | ||
| config = load_config(Path(os.environ["CONFIG"])) | ||
| repo = config.get("repo", {}) | ||
| repo_url = repo.get("url", "https://github.com/InfiniTensor/InfiniOps.git") | ||
| branch = env_opt("BRANCH") or repo.get("branch", "master") | ||
| platform = "${{ matrix.platform }}" | ||
| job_key = "${{ matrix.id }}" | ||
| if not platform: | ||
| print("error: matrix.platform is empty", file=sys.stderr) | ||
| sys.exit(1) | ||
| jobs = config.get("jobs", {}) | ||
| if not jobs: | ||
| print("error: no jobs in config", file=sys.stderr) | ||
| sys.exit(1) | ||
| if job_key not in jobs: | ||
| print(f"error: job {job_key!r} not in config", file=sys.stderr) | ||
| sys.exit(1) | ||
| job = jobs[job_key] | ||
| all_stages = job.get("stages", []) | ||
| stage_filter = env_opt("STAGE") | ||
| if stage_filter: | ||
| stages = [s for s in all_stages if s["name"] == stage_filter] | ||
| if not stages: | ||
| print( | ||
| f"error: stage {stage_filter!r} not found in {job_key}", | ||
| file=sys.stderr, | ||
| ) | ||
| sys.exit(1) | ||
| else: | ||
| stages = all_stages | ||
| test_override = env_opt("TEST") | ||
| if test_override: | ||
| stages = [ | ||
| { | ||
| **s, | ||
| "run": apply_test_override(s.get("run", ""), test_override), | ||
| } | ||
| for s in stages | ||
| ] | ||
| commit = get_git_commit() | ||
| results_base = env_opt("RESULT_DIR") | ||
| if results_base is None: | ||
| results_base = Path("ci-results") | ||
| else: | ||
| results_base = Path(results_base) | ||
| results_dir = build_results_dir(results_base, platform, stages, commit) | ||
| local_path = Path.cwd().resolve() if env_opt("LOCAL") else None | ||
| docker_args = build_docker_args( | ||
| config, | ||
| job_key, | ||
| repo_url, | ||
| branch, | ||
| stages, | ||
| "/workspace", | ||
| env_opt("IMAGE_TAG"), | ||
| gpu_id_override=os.environ.get("GPU_ID") or None, | ||
| results_dir=results_dir, | ||
| local_path=local_path, | ||
| ) | ||
| github_env = os.environ.get("GITHUB_ENV") | ||
| if not github_env: | ||
| print("error: GITHUB_ENV is not set", file=sys.stderr) | ||
| sys.exit(1) | ||
| image_ref = resolve_image( | ||
| config, | ||
| job.get("platform", platform), | ||
| env_opt("IMAGE_TAG") or job.get("image", "latest"), | ||
| ) | ||
| docker_args_text = shlex.join(docker_args) | ||
| delimiter = f"EOF_{uuid.uuid4().hex}" | ||
| with open(github_env, "a", encoding="utf-8") as f: | ||
| # DOCKER_ARGS contains an embedded multiline bash script argument; | ||
| # write it using the multiline GITHUB_ENV syntax. | ||
| f.write(f"DOCKER_ARGS<<{delimiter}\n") | ||
| f.write(docker_args_text + "\n") | ||
| f.write(f"{delimiter}\n") | ||
| f.write(f"RESULT_DIR={results_dir}\n") | ||
| f.write(f"IMAGE_TAG={image_ref}\n") | ||
| PY | ||
| - name: Trigger ${{ matrix.platform }} XPU Unit Test Task | ||
| env: | ||
| ${{ matrix.job_env }} | ||
| run: | | ||
| set -e | ||
| echo "==> running job: ${{ matrix.job_name }}" | ||
| set -m | ||
| if [ -n "${RESULT_DIR:-}" ] && [ "${RESULT_DIR}" != "None" ]; then | ||
| mkdir -p "${RESULT_DIR}" | ||
| fi | ||
| src_dir=$(pwd) | ||
| echo "src_dir: ${src_dir}" | ||
| cd /data/shared/limingge/artifacts | ||
| if [ ! -d "CI_${{ matrix.platform }}_test" ]; then | ||
| mkdir -p "CI_${{ matrix.platform }}_test" | ||
| fi | ||
| cd CI_${{ matrix.platform }}_test | ||
| mkdir -p ${{ github.run_id }}_${{ matrix.id }} | ||
| cd ${{ github.run_id }}_${{ matrix.id }} | ||
| cp ${src_dir}/.ci/daemon.sh . | ||
| chmod a+x ./daemon.sh | ||
| PLATFORM=${{ matrix.platform }} | ||
| echo "**************************************************************************************" | ||
| echo "${PLATFORM^}" | ||
| echo "Unit" | ||
| echo "InfiniTensor" | ||
| echo "${DOCKER_ARGS}" | ||
| echo "${{ github.run_id }}" | ||
| echo "${IMAGE_TAG}" | ||
| echo "**************************************************************************************" | ||
| exec ./daemon.sh \ | ||
| "${PLATFORM^}" \ | ||
| "Unit" \ | ||
| "InfiniTensor" \ | ||
| "None" \ | ||
| "${DOCKER_ARGS}" \ | ||
| ${{ github.run_id }} \ | ||
| "${IMAGE_TAG}" | ||
