Skip to content

Commit 1015a20

Browse files
adibarraadibarra
authored
[WIP] Add CI job priority system (#2217)
* Prototype priority-aware CI scheduling * Reserve p0 for Core skip queue * Make CI priorities higher-first * Keep CI priorities nonnegative * Reuse REPO_PAT for skip queue authorization * Make CI priorities fully automatic * Address priority scheduler review findings * Drop priority label p prefix * Classify patchwork sweeps with Fable * Classify all priority criteria with Fable * Apply Fable criteria per matrix job * Model priority classification in sweep gating tests * Update sweep dependency validation for priority classification * Combine CI priority and queue labels * Move priority controller to CI tracker * Add priority scheduler canary workflow * Harden CI priority scheduling contracts * Fix priority metadata for agentic eval jobs * Harden reusable benchmark input contracts * Tighten priority classification gate * Fix speedbench priority queue token * Simplify priority classification flow * Classify priority on push sweeps * Update priority workflow structure test * Restore optional run-eval input * Align priority diff with existing workflows * Remove priority config EOF whitespace * Use generic priority classifier naming * Run CI priority tooling with uv * Bootstrap uv for CI priority tooling * Remove priority scheduler canary
1 parent 6a1f7c8 commit 1015a20

13 files changed

Lines changed: 958 additions & 37 deletions

‎.github/workflows/benchmark-multinode-tmpl.yml‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,19 @@ on:
66
runner:
77
required: true
88
type: string
9+
priority:
10+
description: "Higher-is-sooner CI priority score"
11+
required: true
12+
type: string
13+
queue-token:
14+
description: "One-shot queue token generated for this job"
15+
required: true
16+
type: string
17+
skip-queue-pr:
18+
description: "Requested skip_queue PR number"
19+
required: false
20+
type: string
21+
default: ''
922
image:
1023
required: true
1124
type: string
@@ -231,10 +244,30 @@ permissions:
231244

232245
jobs:
233246
benchmark:
234-
runs-on: ${{ inputs.runner }}
247+
runs-on: >-
248+
${{ fromJSON(
249+
vars.PRIORITY_SCHEDULER_ENABLED == 'true' &&
250+
(
251+
inputs.skip-queue-pr != '' &&
252+
format(
253+
'["self-hosted",{0},{1},{2},{3}]',
254+
toJSON(inputs.runner),
255+
toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
256+
toJSON(format('ci-attempt-{0}', github.run_attempt)),
257+
toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr))
258+
) ||
259+
format(
260+
'["self-hosted",{0},{1},{2}]',
261+
toJSON(inputs.runner),
262+
toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
263+
toJSON(format('ci-attempt-{0}', github.run_attempt))
264+
)
265+
) ||
266+
format('[{0}]', toJSON(inputs.runner))
267+
) }}
235268
timeout-minutes: 480
236269
name: >-
237-
${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }}
270+
p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }}
238271
${{ inputs.prefill-num-worker }}P (TP${{ inputs.prefill-tp }}${{ inputs.prefill-pp != '' && inputs.prefill-pp != '1' && format('/PP{0}', inputs.prefill-pp) || '' }}${{ inputs.prefill-dcp-size != '' && inputs.prefill-dcp-size != '1' && format('/DCP{0}', inputs.prefill-dcp-size) || '' }}${{ inputs.prefill-pcp-size != '' && inputs.prefill-pcp-size != '1' && format('/PCP{0}', inputs.prefill-pcp-size) || '' }}${{ inputs.prefill-ep != '' && inputs.prefill-ep != '1' && format('/EP{0}', inputs.prefill-ep) || '' }}${{ inputs.prefill-dp-attn == 'true' && '/DPA' || '' }})
239272
x ${{ inputs.decode-num-worker }}D (TP${{ inputs.decode-tp }}${{ inputs.decode-pp != '' && inputs.decode-pp != '1' && format('/PP{0}', inputs.decode-pp) || '' }}${{ inputs.decode-dcp-size != '' && inputs.decode-dcp-size != '1' && format('/DCP{0}', inputs.decode-dcp-size) || '' }}${{ inputs.decode-pcp-size != '' && inputs.decode-pcp-size != '1' && format('/PCP{0}', inputs.decode-pcp-size) || '' }}${{ inputs.decode-ep != '' && inputs.decode-ep != '1' && format('/EP{0}', inputs.decode-ep) || '' }}${{ inputs.decode-dp-attn == 'true' && '/DPA' || '' }})
240273
${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }}

‎.github/workflows/benchmark-tmpl.yml‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,19 @@ on:
55
runner:
66
required: true
77
type: string
8+
priority:
9+
description: "Higher-is-sooner CI priority score"
10+
required: true
11+
type: string
12+
queue-token:
13+
description: "One-shot queue token generated for this job"
14+
required: true
15+
type: string
16+
skip-queue-pr:
17+
description: "Requested skip_queue PR number"
18+
required: false
19+
type: string
20+
default: ''
821
image:
922
required: true
1023
type: string
@@ -179,10 +192,30 @@ permissions:
179192

180193
jobs:
181194
benchmark:
182-
runs-on: ${{ inputs.runner }}
195+
runs-on: >-
196+
${{ fromJSON(
197+
vars.PRIORITY_SCHEDULER_ENABLED == 'true' &&
198+
(
199+
inputs.skip-queue-pr != '' &&
200+
format(
201+
'["self-hosted",{0},{1},{2},{3}]',
202+
toJSON(inputs.runner),
203+
toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
204+
toJSON(format('ci-attempt-{0}', github.run_attempt)),
205+
toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr))
206+
) ||
207+
format(
208+
'["self-hosted",{0},{1},{2}]',
209+
toJSON(inputs.runner),
210+
toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
211+
toJSON(format('ci-attempt-{0}', github.run_attempt))
212+
)
213+
) ||
214+
format('[{0}]', toJSON(inputs.runner))
215+
) }}
183216
timeout-minutes: 500
184217
name: >-
185-
${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }}
218+
p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }}
186219
TP${{ inputs.tp }}${{ inputs.pp != '' && inputs.pp != '1' && format('/PP{0}', inputs.pp) || '' }}${{ inputs.dcp-size != '' && inputs.dcp-size != '1' && format('/DCP{0}', inputs.dcp-size) || '' }}${{ inputs.pcp-size != '' && inputs.pcp-size != '1' && format('/PCP{0}', inputs.pcp-size) || '' }}${{ inputs.ep != '' && inputs.ep != '1' && format('/EP{0}', inputs.ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }}
187220
${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }}
188221
${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }}

‎.github/workflows/collectivex-sweep.yml‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,16 @@ jobs:
3838
outputs:
3939
matrix: ${{ steps.gen.outputs.matrix }}
4040
n: ${{ steps.gen.outputs.n }}
41+
priority: ${{ steps.score.outputs.priority }}
4142
steps:
4243
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
4344
with: { clean: true, persist-credentials: false }
45+
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
46+
- id: score
47+
run: |
48+
scored=$(printf '%s' '[{"runner":"collectivex","framework":"collectivex"}]' |
49+
uv run --no-project --with pyyaml --python 3.12 utils/ci_priority.py)
50+
echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT"
4451
- id: gen
4552
working-directory: experimental/CollectiveX
4653
env:
@@ -49,6 +56,8 @@ jobs:
4956
INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }}
5057
INPUT_EP_SIZES: ${{ inputs.ep_sizes }}
5158
INPUT_MODES: ${{ inputs.modes }}
59+
RUN_ID: ${{ github.run_id }}
60+
RUN_ATTEMPT: ${{ github.run_attempt }}
5261
run: |
5362
set -euo pipefail
5463
args=(--backend "$INPUT_BACKEND")
@@ -58,12 +67,18 @@ jobs:
5867
[ -n "$INPUT_MODES" ] && args+=(--modes "$INPUT_MODES")
5968
python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
6069
python3 - "$GITHUB_OUTPUT" <<'PY'
70+
import hashlib
6171
import json
72+
import os
6273
import pathlib
6374
import sys
6475
6576
matrix = json.loads(pathlib.Path("matrix_full.json").read_text())
6677
cells = matrix["include"]
78+
for index, cell in enumerate(cells):
79+
canonical = json.dumps(cell, sort_keys=True, separators=(",", ":"))
80+
material = f"{os.environ['RUN_ID']}:{os.environ['RUN_ATTEMPT']}:{index}:{canonical}".encode()
81+
cell["queue-token"] = hashlib.sha256(material).hexdigest()[:32]
6782
slim = {"include": cells}
6883
with open(sys.argv[1], "a", encoding="utf-8") as output:
6984
output.write(f"matrix={json.dumps(slim, separators=(',', ':'))}\n")
@@ -91,7 +106,22 @@ jobs:
91106
# runners (10 was too low when the shared fleet is busy with concurrent sweeps).
92107
max-parallel: 30
93108
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
94-
runs-on: ${{ matrix.sku }}
109+
runs-on: >-
110+
${{ fromJSON(
111+
vars.PRIORITY_SCHEDULER_ENABLED == 'true' &&
112+
format(
113+
'["self-hosted",{0},{1},{2}]',
114+
toJSON(matrix.sku),
115+
toJSON(format(
116+
'ci-job-{0}-{1}',
117+
needs.setup.outputs.priority,
118+
matrix.queue-token
119+
)),
120+
toJSON(format('ci-attempt-{0}', github.run_attempt))
121+
) ||
122+
format('[{0}]', toJSON(matrix.sku))
123+
) }}
124+
name: p${{ needs.setup.outputs.priority }} | ${{ matrix.sku }} ${{ matrix.backend }} shard ${{ matrix.id }}
95125
timeout-minutes: 350
96126
env:
97127
COLLX_BENCH: ${{ matrix.backend }}

‎.github/workflows/e2e-tests.yml‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -84,19 +84,41 @@ jobs:
8484
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
8585
with:
8686
ref: ${{ github.sha }}
87+
- name: Checkout priority scheduler tooling
88+
if: ${{ inputs.ref && inputs.ref != '' }}
89+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
90+
with:
91+
ref: ${{ github.workflow_sha }}
92+
path: .ci-priority
93+
persist-credentials: false
8794

95+
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
8896
- id: get-jobs
97+
env:
98+
PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
8999
run: |
90-
pip install pydantic
91-
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \
100+
CONFIG_JSON=$(uv run --no-project --with pydantic --with pyyaml --python 3.12 \
101+
${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \
92102
${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }})
93-
AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))")
94-
AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))")
95-
MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))")
96-
SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))")
97-
MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))")
98-
EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))")
99-
MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))")
103+
PRIORITY_ROOT="${GITHUB_WORKSPACE}"
104+
if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then
105+
PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority"
106+
fi
107+
score_matrix() {
108+
local family="$1"
109+
uv run --no-project --with pyyaml --python 3.12 "${PRIORITY_ROOT}/utils/ci_priority.py" \
110+
--policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \
111+
--event-name "${{ github.event_name }}" \
112+
--queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}:${family}" \
113+
--labels-json "$PR_LABELS"
114+
}
115+
AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))" | score_matrix agentic)
116+
AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval)
117+
MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))" | score_matrix multi-agentic)
118+
SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single)
119+
MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi)
120+
EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval)
121+
MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval)
100122
echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT
101123
echo "agentic-eval-config=$AGENTIC_EVAL" >> $GITHUB_OUTPUT
102124
echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT
@@ -120,6 +142,8 @@ jobs:
120142
osl: ${{ matrix.config.osl }}
121143
max-model-len: ${{ matrix.config.max-model-len }}
122144
runner: ${{ matrix.config.runner }}
145+
priority: ${{ matrix.config.priority }}
146+
queue-token: ${{ matrix.config['queue-token'] }}
123147
image: ${{ matrix.config.image }}
124148
model: ${{ matrix.config.model }}
125149
model-prefix: ${{ matrix.config.model-prefix }}
@@ -169,6 +193,8 @@ jobs:
169193
osl: ${{ matrix.config.osl }}
170194
max-model-len: ${{ matrix.config.max-model-len }}
171195
runner: ${{ matrix.config.runner }}
196+
priority: ${{ matrix.config.priority }}
197+
queue-token: ${{ matrix.config['queue-token'] }}
172198
image: ${{ matrix.config.image }}
173199
model: ${{ matrix.config.model }}
174200
model-prefix: ${{ matrix.config.model-prefix }}
@@ -218,6 +244,8 @@ jobs:
218244
with:
219245
exp-name: ${{ matrix.config.exp-name }}
220246
runner: ${{ matrix.config.runner }}
247+
priority: ${{ matrix.config.priority }}
248+
queue-token: ${{ matrix.config['queue-token'] }}
221249
image: ${{ matrix.config.image }}
222250
model: ${{ matrix.config.model }}
223251
model-prefix: ${{ matrix.config.model-prefix }}
@@ -241,7 +269,7 @@ jobs:
241269
osl: '0'
242270
max-model-len: '0'
243271
spec-decoding: ${{ matrix.config.spec-decoding }}
244-
disagg: 'false'
272+
disagg: ${{ 'false' }}
245273
run-eval: false
246274
scenario-type: agentic-coding
247275
ref: ${{ inputs.ref }}
@@ -259,6 +287,8 @@ jobs:
259287
with:
260288
exp-name: ${{ matrix.config.exp-name }}
261289
runner: ${{ matrix.config.runner }}
290+
priority: ${{ matrix.config.priority }}
291+
queue-token: ${{ matrix.config['queue-token'] }}
262292
image: ${{ matrix.config.image }}
263293
model: ${{ matrix.config.model }}
264294
model-prefix: ${{ matrix.config.model-prefix }}
@@ -276,7 +306,7 @@ jobs:
276306
osl: '0'
277307
max-model-len: '0'
278308
spec-decoding: 'none'
279-
disagg: 'false'
309+
disagg: ${{ 'false' }}
280310
run-eval: true
281311
eval-only: true
282312
eval-limit: ${{ inputs.eval-limit }}
@@ -300,6 +330,8 @@ jobs:
300330
osl: '0'
301331
max-model-len: '0'
302332
runner: ${{ matrix.config.runner }}
333+
priority: ${{ matrix.config.priority }}
334+
queue-token: ${{ matrix.config['queue-token'] }}
303335
image: ${{ matrix.config.image }}
304336
model: ${{ matrix.config.model }}
305337
model-prefix: ${{ matrix.config.model-prefix }}
@@ -353,6 +385,8 @@ jobs:
353385
osl: ${{ matrix.config.osl }}
354386
max-model-len: ${{ matrix.config.max-model-len }}
355387
runner: ${{ matrix.config.runner }}
388+
priority: ${{ matrix.config.priority }}
389+
queue-token: ${{ matrix.config['queue-token'] }}
356390
image: ${{ matrix.config.image }}
357391
model: ${{ matrix.config.model }}
358392
model-prefix: ${{ matrix.config.model-prefix }}
@@ -388,6 +422,8 @@ jobs:
388422
osl: ${{ matrix.config.osl }}
389423
max-model-len: ${{ matrix.config.max-model-len }}
390424
runner: ${{ matrix.config.runner }}
425+
priority: ${{ matrix.config.priority }}
426+
queue-token: ${{ matrix.config['queue-token'] }}
391427
image: ${{ matrix.config.image }}
392428
model: ${{ matrix.config.model }}
393429
model-prefix: ${{ matrix.config.model-prefix }}

0 commit comments

Comments
 (0)