Skip to content

Commit aa25964

Browse files
r4victorr4victor
authored
Support building staging Docker images (#2664)
* Support building staging Docker images * Allow using staging Docker image
1 parent 1e07524 commit aa25964

13 files changed

Lines changed: 77 additions & 30 deletions

File tree

‎.github/workflows/docker.yml‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,15 @@ on:
88
description: "Docker image version"
99
required: true
1010
staging:
11-
description: "Staging build"
11+
description: "Build staging images"
12+
type: boolean
13+
default: false
14+
prepull_staging:
15+
description: "Pre-pull staging Docker images"
1216
type: boolean
1317
default: false
1418
build_docker:
15-
description: "Build docker images"
19+
description: "Build Docker images"
1620
type: boolean
1721
default: true
1822
build_aws:
@@ -34,7 +38,9 @@ on:
3438

3539
env:
3640
PACKER_VERSION: "1.9.2"
37-
BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }} # staging ? prefix : ''
41+
BUILD_DOCKER_REPO: ${{ inputs.staging && 'base-stgn' || 'base' }}
42+
PREPULL_DOCKER_REPO: ${{ inputs.prepull_staging && 'base-stgn' || 'base' }}
43+
VM_IMAGE_BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }} # staging ? prefix : ''
3844

3945
jobs:
4046
build-docker:
@@ -61,7 +67,7 @@ jobs:
6167
uses: docker/setup-qemu-action@v3
6268
- name: Build and upload to DockerHub
6369
run: |
64-
docker buildx build --platform linux/amd64 --build-arg FLAVOR=${{ matrix.flavor }} --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1${{ matrix.flavor == 'devel' && '-devel' || '' }} -f base/Dockerfile .
70+
docker buildx build --platform linux/amd64 --build-arg FLAVOR=${{ matrix.flavor }} --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/${{ env.BUILD_DOCKER_REPO }}:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1${{ matrix.flavor == 'devel' && '-devel' || '' }} -f base/Dockerfile .
6571
6672
build-aws-images:
6773
needs: build-docker
@@ -85,7 +91,7 @@ jobs:
8591
chmod +x packer
8692
- name: Run packer
8793
run: |
88-
./packer build -var-file=versions.json $PROD_VARS -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX aws-image${{ matrix.variant }}.json
94+
./packer build -var-file=versions.json $PROD_VARS -var image_repo=${{ env.PREPULL_DOCKER_REPO }} -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX aws-image${{ matrix.variant }}.json
8995
env:
9096
PROD_VARS: ${{ !inputs.staging && '-var-file=aws-vars-prod.json' || '' }} # production ? var-file : ''
9197

@@ -118,12 +124,12 @@ jobs:
118124
chmod +x packer
119125
- name: Run packer
120126
run: |
121-
./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX azure-image${{ matrix.variant }}.json
127+
./packer build -var-file=versions.json -var image_repo=${{ env.PREPULL_DOCKER_REPO }} -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX azure-image${{ matrix.variant }}.json
122128
- name: Publish azure image
123129
if: ${{ !inputs.staging }}
124130
run: |
125-
IMAGE_DEFINITION=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
126-
IMAGE_NAME=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
131+
IMAGE_DEFINITION=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
132+
IMAGE_NAME=${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
127133
../publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME
128134
129135
build-gcp-images:
@@ -156,11 +162,11 @@ jobs:
156162
chmod +x packer
157163
- name: Run packer
158164
run: |
159-
./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX gcp-image${{ matrix.variant }}.json
165+
./packer build -var-file=versions.json -var image_repo=${{ env.PREPULL_DOCKER_REPO }} -var image_version=${{ inputs.image_version }} -var build_prefix=$VM_IMAGE_BUILD_PREFIX gcp-image${{ matrix.variant }}.json
160166
- name: Publish images
161167
run: |
162168
IMAGE_VERSION=${IMAGE_VERSION//./-}
163-
gcloud compute images add-iam-policy-binding ${BUILD_PREFIX}dstack${{ matrix.variant }}-$IMAGE_VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser'
169+
gcloud compute images add-iam-policy-binding ${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-$IMAGE_VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser'
164170
env:
165171
IMAGE_VERSION: ${{ inputs.image_version }}
166172

@@ -205,8 +211,9 @@ jobs:
205211
run: |
206212
./packer build \
207213
-var-file=versions.json \
214+
-var image_repo=${{ env.PREPULL_DOCKER_REPO }} \
208215
-var image_version=${{ inputs.image_version }} \
209-
-var build_prefix=$BUILD_PREFIX \
216+
-var build_prefix=$VM_IMAGE_BUILD_PREFIX \
210217
-var oci_compartment_ocid=$OCI_COMPARTMENT \
211218
-var oci_subnet_ocid=$OCI_SUBNET \
212219
-var oci_availability_domain=$OCI_AVAILABILITY_DOMAIN \
@@ -221,14 +228,14 @@ jobs:
221228
if: ${{ !inputs.staging }}
222229
run: |
223230
uv run scripts/oci_image_tools.py copy \
224-
--image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
231+
--image ${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
225232
--from $OCI_REGION \
226233
--compartment $OCI_COMPARTMENT
227234
- name: Publish image in OCI Marketplace
228235
if: ${{ !inputs.staging }}
229236
run: |
230237
uv run scripts/oci_image_tools.py publish \
231-
--image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
238+
--image ${VM_IMAGE_BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
232239
--compartment $OCI_COMPARTMENT \
233240
--version ${{ inputs.image_version }} \
234241
--description "Image for running workloads with dstack - https://dstack.ai/" \

‎scripts/packer/aws-image-cuda.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"build_prefix": "",
1313
"ami_regions": "",
1414
"ami_groups": "",
15+
"image_repo": "",
1516
"image_version": ""
1617
},
1718
"builders": [
@@ -83,7 +84,10 @@
8384
},
8485
{
8586
"type": "shell",
86-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
87+
"environment_vars": [
88+
"IMAGE_REPO={{user `image_repo`}}",
89+
"IMAGE_VERSION={{user `image_version`}}"
90+
],
8791
"script": "provisioners/pull-docker-images.sh"
8892
}
8993
]

‎scripts/packer/aws-image.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"build_prefix": "",
1212
"ami_regions": "",
1313
"ami_groups": "",
14+
"image_repo": "",
1415
"image_version": ""
1516
},
1617
"builders": [
@@ -73,7 +74,10 @@
7374
},
7475
{
7576
"type": "shell",
76-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
77+
"environment_vars": [
78+
"IMAGE_REPO={{user `image_repo`}}",
79+
"IMAGE_VERSION={{user `image_version`}}"
80+
],
7781
"script": "provisioners/pull-docker-images.sh"
7882
}
7983
]

‎scripts/packer/azure-image-cuda.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"build_prefix": "",
1111
"docker_version": "",
1212
"cuda_drivers_version": "",
13+
"image_repo": "",
1314
"image_version": ""
1415
},
1516
"builders": [{
@@ -74,7 +75,10 @@
7475
},
7576
{
7677
"type": "shell",
77-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
78+
"environment_vars": [
79+
"IMAGE_REPO={{user `image_repo`}}",
80+
"IMAGE_VERSION={{user `image_version`}}"
81+
],
7882
"script": "provisioners/pull-docker-images.sh"
7983
},
8084
{

‎scripts/packer/azure-image-grid.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"azure_vm_size": "Standard_DS1_v2",
1010
"build_prefix": "",
1111
"docker_version": "",
12+
"image_repo": "",
1213
"image_version": ""
1314
},
1415
"builders": [{
@@ -72,7 +73,10 @@
7273
},
7374
{
7475
"type": "shell",
75-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
76+
"environment_vars": [
77+
"IMAGE_REPO={{user `image_repo`}}",
78+
"IMAGE_VERSION={{user `image_version`}}"
79+
],
7680
"script": "provisioners/pull-docker-images.sh"
7781
},
7882
{

‎scripts/packer/azure-image.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"azure_vm_size": "Standard_DS1_v2",
1010
"build_prefix": "",
1111
"docker_version": "",
12+
"image_repo": "",
1213
"image_version": ""
1314
},
1415
"builders": [{
@@ -64,7 +65,10 @@
6465
},
6566
{
6667
"type": "shell",
67-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
68+
"environment_vars": [
69+
"IMAGE_REPO={{user `image_repo`}}",
70+
"IMAGE_VERSION={{user `image_version`}}"
71+
],
6872
"script": "provisioners/pull-docker-images.sh"
6973
},
7074
{

‎scripts/packer/gcp-image-cuda.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"build_prefix": "",
44
"docker_version": "",
55
"cuda_drivers_version": "",
6+
"image_repo": "",
67
"image_version": ""
78
},
89
"builders": [
@@ -58,7 +59,10 @@
5859
},
5960
{
6061
"type": "shell",
61-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
62+
"environment_vars": [
63+
"IMAGE_REPO={{user `image_repo`}}",
64+
"IMAGE_VERSION={{user `image_version`}}"
65+
],
6266
"script": "provisioners/pull-docker-images.sh"
6367
}
6468
]

‎scripts/packer/gcp-image.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"variables": {
33
"build_prefix": "",
44
"docker_version": "",
5+
"image_repo": "",
56
"image_version": ""
67
},
78
"builders": [
@@ -48,7 +49,10 @@
4849
},
4950
{
5051
"type": "shell",
51-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
52+
"environment_vars": [
53+
"IMAGE_REPO={{user `image_repo`}}",
54+
"IMAGE_VERSION={{user `image_version`}}"
55+
],
5256
"script": "provisioners/pull-docker-images.sh"
5357
}
5458
]

‎scripts/packer/oci-image-cuda.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"build_prefix": "",
44
"docker_version": "",
55
"cuda_drivers_version": "",
6+
"image_repo": "",
67
"image_version": "",
78
"oci_availability_domain": "",
89
"oci_compartment_ocid": "",
@@ -67,7 +68,10 @@
6768
},
6869
{
6970
"type": "shell",
70-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
71+
"environment_vars": [
72+
"IMAGE_REPO={{user `image_repo`}}",
73+
"IMAGE_VERSION={{user `image_version`}}"
74+
],
7175
"script": "provisioners/pull-docker-images.sh"
7276
}
7377
]

‎scripts/packer/oci-image.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"variables": {
33
"build_prefix": "",
44
"docker_version": "",
5+
"image_repo": "",
56
"image_version": "",
67
"oci_availability_domain": "",
78
"oci_compartment_ocid": "",
@@ -57,7 +58,10 @@
5758
},
5859
{
5960
"type": "shell",
60-
"environment_vars": ["IMAGE_VERSION={{user `image_version`}}"],
61+
"environment_vars": [
62+
"IMAGE_REPO={{user `image_repo`}}",
63+
"IMAGE_VERSION={{user `image_version`}}"
64+
],
6165
"script": "provisioners/pull-docker-images.sh"
6266
}
6367
]

0 commit comments

Comments
 (0)