Skip to content

Commit 4bfdc63

Browse files
jvstmejvstme
committed
CI refactoring
- Split GitHub Actions workflows into reusable workflows to avoid duplication. - Split some jobs into two stages: building an artifact and uploading it. This allows to upload artifacts only after all were successfully built. Also allows to test building artifacts in PRs from forks, where credentials for uploading are not available.
1 parent 0d0e654 commit 4bfdc63

File tree

9 files changed

+508
-552
lines changed

9 files changed

+508
-552
lines changed

‎.github/workflows/build-artifacts.yml‎

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
name: Build Artifacts
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
version:
7+
type: string
8+
required: true
9+
staging:
10+
type: boolean
11+
required: true
12+
go-integration-tests:
13+
type: boolean
14+
required: true
15+
16+
jobs:
17+
code-lint:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v4
21+
- name: Set up uv
22+
uses: astral-sh/setup-uv@v5
23+
with:
24+
python-version: 3.11
25+
- run: uv tool install pre-commit
26+
- run: pre-commit run -a --show-diff-on-failure
27+
28+
frontend-build:
29+
runs-on: ubuntu-latest
30+
defaults:
31+
run:
32+
working-directory: frontend
33+
steps:
34+
- uses: actions/checkout@v4
35+
- name: Restore cached build
36+
id: cache-build
37+
uses: actions/cache@v4
38+
with:
39+
path: frontend/build
40+
key: frontend-build-${{ hashFiles('frontend/**') }}
41+
restore-keys: |
42+
frontend-build-
43+
- name: Set up Node
44+
if: steps.cache-build.outputs.cache-hit != 'true'
45+
uses: actions/setup-node@v4
46+
with:
47+
node-version: 18
48+
- name: Install packages
49+
if: steps.cache-build.outputs.cache-hit != 'true'
50+
run: npm ci
51+
- name: Build dist
52+
if: steps.cache-build.outputs.cache-hit != 'true'
53+
run: npm run build
54+
- name: Upload dist
55+
uses: actions/upload-artifact@v4
56+
with:
57+
name: frontend-build
58+
path: frontend/build
59+
retention-days: 1
60+
61+
python-test:
62+
needs: [code-lint, frontend-build]
63+
runs-on: ${{ matrix.os }}
64+
strategy:
65+
matrix:
66+
os: [macos-latest, ubuntu-latest, windows-latest]
67+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
68+
steps:
69+
- uses: actions/checkout@v4
70+
- name: Set up Python ${{ matrix.python-version }}
71+
uses: astral-sh/setup-uv@v5
72+
with:
73+
python-version: ${{ matrix.python-version }}
74+
- name: Install dependencies
75+
run: uv sync --all-extras
76+
- name: Download frontend build
77+
uses: actions/download-artifact@v4
78+
with:
79+
name: frontend-build
80+
path: src/dstack/_internal/server/statics
81+
- name: Run pytest on POSIX
82+
if: matrix.os != 'windows-latest'
83+
# Skip Postgres tests on macos since macos runner doesn't have Docker.
84+
run: |
85+
RUNPOSTGRES=""
86+
if [ "${{ matrix.os }}" != "macos-latest" ]; then
87+
RUNPOSTGRES="--runpostgres"
88+
fi
89+
uv run pytest -n auto src/tests --runui $RUNPOSTGRES
90+
- name: Run pytest on Windows
91+
if: matrix.os == 'windows-latest'
92+
run: |
93+
uv run pytest -n auto src/tests --runui --runpostgres
94+
95+
runner-test:
96+
defaults:
97+
run:
98+
working-directory: runner
99+
runs-on: ${{ matrix.os }}
100+
strategy:
101+
matrix:
102+
os: [ubuntu-latest, macos-latest]
103+
steps:
104+
- uses: actions/checkout@v4
105+
- name: Set up Go
106+
uses: actions/setup-go@v5
107+
with:
108+
go-version-file: runner/go.mod
109+
cache-dependency-path: runner/go.sum
110+
- name: Check if go.mod and go.sum are up-to-date
111+
run: go mod tidy -diff
112+
- name: Run golangci-lint
113+
uses: golangci/golangci-lint-action@v6
114+
with:
115+
version: v1.62.0 # Should match .pre-commit-config.yaml
116+
args: --timeout=20m
117+
working-directory: runner
118+
- name: Test
119+
# Only run slow tests if requested by workflow call inputs.
120+
run: |
121+
SHORT="-short"
122+
if [[ "${{ inputs.go-integration-tests }}" == "true" ]]; then
123+
SHORT=""
124+
fi
125+
# Skip failing integration tests on macOS for release builds.
126+
# TODO: https://github.com/dstackai/dstack/issues/3005
127+
if [[ "${{ inputs.staging }}" == "false" && "${{ startsWith(matrix.os, 'macos') }}" == "true" ]]; then
128+
SHORT="-short"
129+
fi
130+
go version
131+
go fmt $(go list ./... | grep -v /vendor/)
132+
go vet $(go list ./... | grep -v /vendor/)
133+
go test $SHORT -race $(go list ./... | grep -v /vendor/)
134+
135+
runner-compile:
136+
needs: [runner-test]
137+
defaults:
138+
run:
139+
working-directory: runner
140+
env:
141+
REPO_NAME: github.com/dstackai/dstack
142+
strategy:
143+
matrix:
144+
include:
145+
- { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" }
146+
- { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" }
147+
runs-on: ${{ matrix.runs-on }}
148+
steps:
149+
- uses: actions/checkout@v4
150+
- name: Set up Go
151+
uses: actions/setup-go@v5
152+
with:
153+
go-version-file: runner/go.mod
154+
cache-dependency-path: runner/go.sum
155+
- name: build
156+
env:
157+
GOOS: ${{ matrix.goos }}
158+
GOARCH: ${{ matrix.goarch }}
159+
run: |
160+
CGO_ENABLED=0 go build -ldflags "-X 'main.Version=${{ inputs.version }}' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner
161+
CGO_ENABLED=1 go build -ldflags "-X 'main.Version=${{ inputs.version }}'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim
162+
- uses: actions/upload-artifact@v4
163+
with:
164+
name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
165+
path: |
166+
runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
167+
runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}
168+
retention-days: 1
169+
170+
gateway-build:
171+
runs-on: ubuntu-latest
172+
steps:
173+
- uses: actions/checkout@v4
174+
- name: Set up uv
175+
uses: astral-sh/setup-uv@v5
176+
with:
177+
python-version: 3.11
178+
- name: Build package
179+
working-directory: gateway
180+
run: |
181+
echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py
182+
# TODO: depend on a specific dstackai/dstack commit for staging builds?
183+
if [[ "${{ inputs.staging }}" == "false" ]]; then
184+
sed \
185+
-i.old \
186+
"s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ inputs.version }}|" \
187+
pyproject.toml
188+
diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1
189+
fi
190+
uv build
191+
- uses: actions/upload-artifact@v4
192+
with:
193+
name: dstack-gateway
194+
path: gateway/dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl
195+
retention-days: 1
196+
197+
python-build:
198+
needs: [code-lint, frontend-build]
199+
runs-on: ubuntu-latest
200+
steps:
201+
- uses: actions/checkout@v4
202+
- name: Set up uv
203+
uses: astral-sh/setup-uv@v5
204+
with:
205+
python-version: 3.11
206+
- name: Download frontend build
207+
uses: actions/download-artifact@v4
208+
with:
209+
name: frontend-build
210+
path: src/dstack/_internal/server/statics
211+
- name: Build dstack Python package
212+
# TODO: set __version__ to inputs.version regardless of inputs.staging,
213+
# so that staging builds are also tied to a specific runner and gateway version.
214+
# May require changing how dstack handles __version__.
215+
run: |
216+
if [[ "${{ inputs.staging }}" == "true" ]]; then
217+
VERSION=0.0.0
218+
IS_RELEASE=False
219+
else
220+
VERSION=${{ inputs.version }}
221+
IS_RELEASE=True
222+
fi
223+
BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ")
224+
BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ")
225+
echo "__version__ = \"$VERSION\"" > src/dstack/version.py
226+
echo "__is_release__ = $IS_RELEASE" >> src/dstack/version.py
227+
echo $BASE_IMAGE >> src/dstack/version.py
228+
echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py
229+
cp README.md src
230+
uv build
231+
- uses: actions/upload-artifact@v4
232+
with:
233+
name: python-build
234+
path: dist
235+
retention-days: 1
236+
237+
generate-json-schema:
238+
needs: [code-lint]
239+
runs-on: ubuntu-latest
240+
steps:
241+
- uses: actions/checkout@v4
242+
- uses: astral-sh/setup-uv@v5
243+
with:
244+
python-version: 3.11
245+
- name: Install dstack
246+
run: uv sync
247+
- name: Generate json schema
248+
run: |
249+
mkdir /tmp/json-schemas
250+
uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json
251+
uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > /tmp/json-schemas/profiles.json
252+
- uses: actions/upload-artifact@v4
253+
with:
254+
name: json-schemas
255+
path: /tmp/json-schemas
256+
retention-days: 1

‎.github/workflows/build-docs.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: Build Docs
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
release_tag:
7+
type: string
8+
required: false
9+
10+
jobs:
11+
build-docs:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: actions/checkout@v4
15+
- uses: astral-sh/setup-uv@v5
16+
with:
17+
python-version: 3.11
18+
- name: Install dstack
19+
run: |
20+
uv pip install examples/plugins/example_plugin_server
21+
if [ -n "${{ inputs.release_tag }}" ]; then
22+
uv pip install "dstack[server]==${{ inputs.release_tag }}"
23+
else
24+
uv pip install -e '.[server]'
25+
fi
26+
- name: Build
27+
run: |
28+
uv pip install pillow cairosvg
29+
sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev
30+
uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade
31+
uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
32+
uv run mkdocs build -s
33+
- uses: actions/upload-artifact@v4
34+
with:
35+
name: site
36+
path: site
37+
retention-days: 1

0 commit comments

Comments
 (0)