Skip to content

Commit 29db4ab

Browse files
localai-botmudler
localai-bot
and
mudler
authored
fix(ci): shard single-arch backend matrix under GitHub's 256-job limit (#10703)
GitHub Actions refuses to instantiate a matrix that would generate more than 256 jobs. It does so silently: the job hangs forever at "Waiting for pending jobs" and the whole run is marked `failure` while every other job stays green. This is exactly what happened on the v4.6.1 tag build (run 28786533892): the single-arch build matrix had grown to 268 entries, so `backend-jobs-singlearch` (and its downstream merge) never produced a single job, and the release build "failed" with no failing job to point at. The single-arch list is the one that grows unbounded as backends are added, so shard it across a fixed number of matrix jobs (SINGLEARCH_SHARDS=4, ~67 entries each today, headroom to ~1020 backends). Each merge shard `needs:` only its matching build shard, preserving the "merge waits only on its own build" property that keeps slow CUDA/ROCm builds from gating multi-arch manifest assembly. changed-backends.js now emits per-shard matrix/has-* outputs and throws loudly if a shard ever reaches the 256 limit (telling the maintainer to bump SINGLEARCH_SHARDS and add matching job blocks) instead of letting GitHub drop the overflow silently. backend.yml and backend_pr.yml define the four build + four merge shard jobs; multi-arch and darwin groups are untouched. Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
1 parent a6cf67c commit 29db4ab

3 files changed

Lines changed: 383 additions & 37 deletions

File tree

‎.github/workflows/backend.yml‎

Lines changed: 169 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,30 @@ jobs:
3232
if: github.repository == 'mudler/LocalAI'
3333
runs-on: ubuntu-latest
3434
outputs:
35-
matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }}
3635
matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }}
3736
matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }}
3837
merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }}
39-
merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }}
40-
has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }}
4138
has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }}
4239
has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }}
4340
has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }}
44-
has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }}
41+
# Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to
42+
# stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js).
43+
matrix-singlearch-1: ${{ steps.set-matrix.outputs['matrix-singlearch-1'] }}
44+
merge-matrix-singlearch-1: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-1'] }}
45+
has-backends-singlearch-1: ${{ steps.set-matrix.outputs['has-backends-singlearch-1'] }}
46+
has-merges-singlearch-1: ${{ steps.set-matrix.outputs['has-merges-singlearch-1'] }}
47+
matrix-singlearch-2: ${{ steps.set-matrix.outputs['matrix-singlearch-2'] }}
48+
merge-matrix-singlearch-2: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-2'] }}
49+
has-backends-singlearch-2: ${{ steps.set-matrix.outputs['has-backends-singlearch-2'] }}
50+
has-merges-singlearch-2: ${{ steps.set-matrix.outputs['has-merges-singlearch-2'] }}
51+
matrix-singlearch-3: ${{ steps.set-matrix.outputs['matrix-singlearch-3'] }}
52+
merge-matrix-singlearch-3: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-3'] }}
53+
has-backends-singlearch-3: ${{ steps.set-matrix.outputs['has-backends-singlearch-3'] }}
54+
has-merges-singlearch-3: ${{ steps.set-matrix.outputs['has-merges-singlearch-3'] }}
55+
matrix-singlearch-4: ${{ steps.set-matrix.outputs['matrix-singlearch-4'] }}
56+
merge-matrix-singlearch-4: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-4'] }}
57+
has-backends-singlearch-4: ${{ steps.set-matrix.outputs['has-backends-singlearch-4'] }}
58+
has-merges-singlearch-4: ${{ steps.set-matrix.outputs['has-merges-singlearch-4'] }}
4559
steps:
4660
- name: Checkout repository
4761
uses: actions/checkout@v7
@@ -109,9 +123,9 @@ jobs:
109123
# take their full ~6h cold without blocking manifest assembly for the
110124
# multi-arch backends whose per-arch digests would otherwise sit untagged
111125
# on quay long enough to be GC'd.
112-
backend-jobs-singlearch:
126+
backend-jobs-singlearch-1:
113127
needs: generate-matrix
114-
if: needs.generate-matrix.outputs['has-backends-singlearch'] == 'true'
128+
if: needs.generate-matrix.outputs['has-backends-singlearch-1'] == 'true'
115129
uses: ./.github/workflows/backend_build.yml
116130
with:
117131
tag-latest: ${{ matrix.tag-latest }}
@@ -138,7 +152,100 @@ jobs:
138152
strategy:
139153
fail-fast: false
140154
max-parallel: 8
141-
matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }}
155+
matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-1']) }}
156+
157+
backend-jobs-singlearch-2:
158+
needs: generate-matrix
159+
if: needs.generate-matrix.outputs['has-backends-singlearch-2'] == 'true'
160+
uses: ./.github/workflows/backend_build.yml
161+
with:
162+
tag-latest: ${{ matrix.tag-latest }}
163+
tag-suffix: ${{ matrix.tag-suffix }}
164+
build-type: ${{ matrix.build-type }}
165+
cuda-major-version: ${{ matrix.cuda-major-version }}
166+
cuda-minor-version: ${{ matrix.cuda-minor-version }}
167+
platforms: ${{ matrix.platforms }}
168+
platform-tag: ${{ matrix.platform-tag || '' }}
169+
runs-on: ${{ matrix.runs-on }}
170+
builder-base-image: ${{ matrix.builder-base-image || '' }}
171+
base-image: ${{ matrix.base-image }}
172+
backend: ${{ matrix.backend }}
173+
dockerfile: ${{ matrix.dockerfile }}
174+
skip-drivers: ${{ matrix.skip-drivers }}
175+
context: ${{ matrix.context }}
176+
ubuntu-version: ${{ matrix.ubuntu-version }}
177+
amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
178+
secrets:
179+
dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
180+
dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
181+
quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
182+
quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
183+
strategy:
184+
fail-fast: false
185+
max-parallel: 8
186+
matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-2']) }}
187+
188+
backend-jobs-singlearch-3:
189+
needs: generate-matrix
190+
if: needs.generate-matrix.outputs['has-backends-singlearch-3'] == 'true'
191+
uses: ./.github/workflows/backend_build.yml
192+
with:
193+
tag-latest: ${{ matrix.tag-latest }}
194+
tag-suffix: ${{ matrix.tag-suffix }}
195+
build-type: ${{ matrix.build-type }}
196+
cuda-major-version: ${{ matrix.cuda-major-version }}
197+
cuda-minor-version: ${{ matrix.cuda-minor-version }}
198+
platforms: ${{ matrix.platforms }}
199+
platform-tag: ${{ matrix.platform-tag || '' }}
200+
runs-on: ${{ matrix.runs-on }}
201+
builder-base-image: ${{ matrix.builder-base-image || '' }}
202+
base-image: ${{ matrix.base-image }}
203+
backend: ${{ matrix.backend }}
204+
dockerfile: ${{ matrix.dockerfile }}
205+
skip-drivers: ${{ matrix.skip-drivers }}
206+
context: ${{ matrix.context }}
207+
ubuntu-version: ${{ matrix.ubuntu-version }}
208+
amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
209+
secrets:
210+
dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
211+
dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
212+
quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
213+
quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
214+
strategy:
215+
fail-fast: false
216+
max-parallel: 8
217+
matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-3']) }}
218+
219+
backend-jobs-singlearch-4:
220+
needs: generate-matrix
221+
if: needs.generate-matrix.outputs['has-backends-singlearch-4'] == 'true'
222+
uses: ./.github/workflows/backend_build.yml
223+
with:
224+
tag-latest: ${{ matrix.tag-latest }}
225+
tag-suffix: ${{ matrix.tag-suffix }}
226+
build-type: ${{ matrix.build-type }}
227+
cuda-major-version: ${{ matrix.cuda-major-version }}
228+
cuda-minor-version: ${{ matrix.cuda-minor-version }}
229+
platforms: ${{ matrix.platforms }}
230+
platform-tag: ${{ matrix.platform-tag || '' }}
231+
runs-on: ${{ matrix.runs-on }}
232+
builder-base-image: ${{ matrix.builder-base-image || '' }}
233+
base-image: ${{ matrix.base-image }}
234+
backend: ${{ matrix.backend }}
235+
dockerfile: ${{ matrix.dockerfile }}
236+
skip-drivers: ${{ matrix.skip-drivers }}
237+
context: ${{ matrix.context }}
238+
ubuntu-version: ${{ matrix.ubuntu-version }}
239+
amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
240+
secrets:
241+
dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
242+
dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
243+
quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
244+
quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
245+
strategy:
246+
fail-fast: false
247+
max-parallel: 8
248+
matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-4']) }}
142249

143250
# Apply tags to per-arch digests via `imagetools create`. Split into two
144251
# jobs that mirror the build split so each merge waits ONLY on its
@@ -174,10 +281,60 @@ jobs:
174281
fail-fast: false
175282
matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }}
176283

177-
backend-merge-jobs-singlearch:
178-
needs: [generate-matrix, backend-jobs-singlearch]
179-
# See note on backend-merge-jobs-multiarch above for !cancelled().
180-
if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch'] == 'true' }}
284+
# One merge shard per build shard: backend-merge-jobs-singlearch-<n> needs only
285+
# backend-jobs-singlearch-<n>, preserving the "merge waits only on its own
286+
# build" property while staying under the 256-jobs-per-matrix limit.
287+
backend-merge-jobs-singlearch-1:
288+
needs: [generate-matrix, backend-jobs-singlearch-1]
289+
if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-1'] == 'true' }}
290+
uses: ./.github/workflows/backend_merge.yml
291+
with:
292+
tag-latest: ${{ matrix.tag-latest }}
293+
tag-suffix: ${{ matrix.tag-suffix }}
294+
secrets:
295+
dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
296+
dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
297+
quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
298+
quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
299+
strategy:
300+
fail-fast: false
301+
matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-1']) }}
302+
303+
backend-merge-jobs-singlearch-2:
304+
needs: [generate-matrix, backend-jobs-singlearch-2]
305+
if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-2'] == 'true' }}
306+
uses: ./.github/workflows/backend_merge.yml
307+
with:
308+
tag-latest: ${{ matrix.tag-latest }}
309+
tag-suffix: ${{ matrix.tag-suffix }}
310+
secrets:
311+
dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
312+
dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
313+
quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
314+
quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
315+
strategy:
316+
fail-fast: false
317+
matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-2']) }}
318+
319+
backend-merge-jobs-singlearch-3:
320+
needs: [generate-matrix, backend-jobs-singlearch-3]
321+
if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-3'] == 'true' }}
322+
uses: ./.github/workflows/backend_merge.yml
323+
with:
324+
tag-latest: ${{ matrix.tag-latest }}
325+
tag-suffix: ${{ matrix.tag-suffix }}
326+
secrets:
327+
dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
328+
dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
329+
quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
330+
quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
331+
strategy:
332+
fail-fast: false
333+
matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-3']) }}
334+
335+
backend-merge-jobs-singlearch-4:
336+
needs: [generate-matrix, backend-jobs-singlearch-4]
337+
if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-4'] == 'true' }}
181338
uses: ./.github/workflows/backend_merge.yml
182339
with:
183340
tag-latest: ${{ matrix.tag-latest }}
@@ -189,7 +346,7 @@ jobs:
189346
quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
190347
strategy:
191348
fail-fast: false
192-
matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }}
349+
matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-4']) }}
193350

194351
backend-jobs-darwin:
195352
needs: generate-matrix

0 commit comments

Comments
 (0)