Skip to content

Commit 620ac59

Browse files
kohtalakohtala
committed
Split build of architectures to different jobs
The runner ran out of memory building all architectures in the same build.
1 parent f3a65e3 commit 620ac59

1 file changed

Lines changed: 91 additions & 31 deletions

File tree

‎.github/workflows/main.yml‎

Lines changed: 91 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -14,33 +14,29 @@ on:
1414
# min hour dom month dow
1515
- cron: '42 3 2 * *'
1616
workflow_dispatch:
17+
1718
permissions:
18-
contents: write # For the tag creation
19+
contents: read
20+
1921
jobs:
2022
build:
21-
runs-on: ubuntu-latest
23+
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
2224
strategy:
2325
matrix:
24-
include:
25-
# Check torch + torchvision compatibility from https://github.com/pytorch/vision?tab=readme-ov-file#installation
26-
- torch: '2.9.1'
27-
python: '3.14.2'
28-
# We need to use project index for CUDA on aarch64 as pypi wheels are for CPU only
29-
index: cu126
30-
platforms: linux/amd64,linux/arm64
31-
constraints: constraints-2.9.1.txt
32-
33-
- torch: '2.9.1'
26+
torch: ['2.9.1']
27+
python: ['3.14.2', '3.14.2-slim']
28+
index: [cu126,cpu]
29+
platform: [linux/amd64, linux/arm64]
30+
exclude:
31+
- index: cpu
32+
platform: linux/arm64
33+
- index: cpu
3434
python: '3.14.2-slim'
35-
index: cu126
36-
platforms: linux/amd64,linux/arm64
37-
constraints: constraints-2.9.1.txt
38-
39-
- torch: '2.9.1'
40-
python: '3.14.2'
41-
index: cpu
42-
platforms: linux/amd64
43-
constraints: constraints-2.9.1.txt
35+
permissions:
36+
contents: read
37+
id-token: write # Needed for actions/attest-build-provenance build predicate
38+
attestations: write # Needed for actions/attest-build-provenance attestation upload to repository
39+
artifact-metadata: write # Needed for actions/attest-build-provenance artifact metadata storage records
4440
steps:
4541
- id: created
4642
run: echo "created=$(date --utc +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_OUTPUT
@@ -62,9 +58,12 @@ jobs:
6258
# Tags have limited set of valid character, '+' not included
6359
# https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests
6460
run: |
65-
echo "tag=$(echo -n "${{ matrix.torch }}${{ matrix.index && format('-{0}', matrix.index) || '' }}-${{ matrix.python }}" | tr -c 'a-zA-Z0-9._-' '[-*]')" >> $GITHUB_OUTPUT
61+
echo "tag=$(echo -n "${{ matrix.torch }}${{ matrix.index && format('-{0}', matrix.index) || '' }}-${{ matrix.python }}${PLATFORM+-}${PLATFORM#*/}" | tr -c 'a-zA-Z0-9._-' '[-*]')" >> $GITHUB_OUTPUT
62+
env:
63+
PLATFORM: ${{ matrix.index == 'cpu' && '' || matrix.platform }}
6664

6765
- uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
66+
id: build-push
6867
with:
6968
context: .
7069
push: ${{ github.ref == 'refs/heads/main' }}
@@ -77,15 +76,82 @@ jobs:
7776
TORCH_WHEEL_SOURCE=${{ matrix.torch-wheel-tag && format('{0}/torch-wheels:{1}', vars.DOCKERHUB_USERNAME, matrix.torch-wheel-tag) || 'scratch'}}
7877
TORCHVISION_WHEEL_SOURCE=${{ matrix.torchvision && format('{0}/torchvision-wheels:{1}', vars.DOCKERHUB_USERNAME, matrix.torchvision) || 'scratch'}}
7978
EXTRA_INDEX_URL=${{ matrix.extra-index-url || format('https://download.pytorch.org/whl/{0}/', matrix.index) }}
80-
${{ matrix.constraints && format('CONSTRAINTS={0}', matrix.constraints) || '' }}
79+
CONSTRAINTS=constraints-${{ matrix.torch }}.txt
8180
# TODO add the latest tag, maybe somehow via docker/metadata-action
8281
tags: |
8382
${{ vars.DOCKERHUB_USERNAME }}/python-torch:${{ steps.build-tag.outputs.tag }}
8483
# https://docs.docker.com/build/ci/github-actions/cache/
8584
cache-from: type=gha,scope=${{ matrix.torch }}-${{ matrix.python }}
8685
cache-to: type=gha,scope=${{ matrix.torch }}-${{ matrix.python }},mode=max
87-
platforms: ${{ matrix.platforms || 'linux/amd64,linux/arm64' }}
86+
platforms: ${{ matrix.platform }}
87+
88+
- name: Create attestation for the image
89+
uses: actions/attest-build-provenance@v3
90+
with:
91+
subject-name: ${{ vars.DOCKERHUB_USERNAME }}/python-torch:${{ steps.build-tag.outputs.tag }}
92+
subject-digest: ${{ steps.build-push.outputs.digest }}
93+
push-to-registry: true
94+
95+
merge:
96+
needs: build
97+
runs-on: ubuntu-latest
98+
strategy:
99+
matrix:
100+
torch: ['2.9.1']
101+
python: ['3.14.2', '3.14.2-slim']
102+
index: [cu126]
103+
permissions:
104+
id-token: write # Needed for actions/attest-build-provenance build predicate
105+
attestations: write # Needed for actions/attest-build-provenance attestation upload to repository
106+
artifact-metadata: write # Needed for actions/attest-build-provenance artifact metadata storage records
107+
steps:
108+
- name: Login to Docker Hub
109+
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
110+
with:
111+
username: ${{ secrets.DOCKERHUB_USERNAME }}
112+
password: ${{ secrets.DOCKERHUB_TOKEN }}
113+
114+
- name: Merge platform images using manifest tool
115+
id: manifest
116+
run: |
117+
# This must match the build job tag generation
118+
tag="$(echo -n "${{ matrix.torch }}${{ matrix.index && format('-{0}', matrix.index) || '' }}-${{ matrix.python }}" | tr -c 'a-zA-Z0-9._-' '[-*]')"
119+
echo "tag=${tag}" >> $GITHUB_OUTPUT
120+
name="$DOCKERHUB_USERNAME/python-torch:${tag}"
121+
echo "name=${name}" >> $GITHUB_OUTPUT
122+
docker buildx imagetools create \
123+
--tag $name \
124+
${name}-amd64 \
125+
${name}-arm64
126+
# Unfortunately we don't get the digest directly so we need to query it
127+
DIGEST=$(docker buildx imagetools inspect "${name}" --format '{{ print .Manifest.Digest }}')
128+
# Ensure the images bundled behind the digest about to be attested still have valid attestations
129+
for digest in $(docker buildx imagetools inspect "$DOCKERHUB_USERNAME/python-torch@$DIGEST" --format '{{ range .Manifest.Manifests }}{{ .Digest }} {{ end }}')
130+
do
131+
gh attestation verify oci://$DOCKERHUB_USERNAME/python-torch@${digest} \
132+
--repo $GITHUB_REPOSITORY \
133+
--signer-workflow $GITHUB_WORKFLOW_REF \
134+
--source-digest $GITHUB_SHA
135+
done
136+
echo "digest=${DIGEST}" >> $GITHUB_OUTPUT
137+
env:
138+
DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }}
139+
140+
- name: Create attestation for the manifest
141+
uses: actions/attest-build-provenance@v3
142+
with:
143+
subject-name: ${{ steps.manifest.outputs.name }}
144+
subject-digest: ${{ steps.manifest.outputs.digest }}
145+
push-to-registry: true
88146

147+
description:
148+
needs: merge
149+
# https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md
150+
runs-on: ubuntu-slim
151+
if: github.ref == 'refs/heads/main'
152+
permissions:
153+
contents: write # For the tag creation
154+
steps:
89155
# With org.opencontainers.image.source pointing to this repository Dockerfile FROM updates in pull requests can be scanned
90156
# if the commits have matching tags with the image.
91157
# https://github.blog/changelog/2023-04-13-dependabot-now-supports-fetching-release-notes-and-changelogs-for-docker-images/
@@ -115,13 +181,7 @@ jobs:
115181
}
116182
}
117183
118-
# Can not use Personal Access Token to update the README. Returns FORBIDDEN.
119-
describe:
120-
needs: build
121-
runs-on: ubuntu-latest
122-
if: github.ref == 'refs/heads/main'
123-
steps:
124-
- uses: actions/checkout@v6
184+
# Can not use Personal Access Token to update the README. Returns FORBIDDEN.
125185
- name: Docker Hub Description
126186
uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0
127187
with:

0 commit comments

Comments
 (0)