Skip to content

Commit 0eaf494

Browse files
jvstmejvstme
authored
[Internal]: CI refactoring (#3006)
1 parent 0d0e654 commit 0eaf494

File tree

9 files changed

+509
-551
lines changed

9 files changed

+509
-551
lines changed

‎.github/workflows/build-artifacts.yml‎

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
name: Build Artifacts
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
version:
7+
type: string
8+
required: true
9+
staging:
10+
type: boolean
11+
required: true
12+
go-integration-tests:
13+
type: boolean
14+
required: true
15+
16+
jobs:
17+
code-lint:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v4
21+
- name: Set up uv
22+
uses: astral-sh/setup-uv@v5
23+
with:
24+
python-version: 3.11
25+
- run: uv tool install pre-commit
26+
- run: pre-commit run -a --show-diff-on-failure
27+
28+
frontend-build:
29+
runs-on: ubuntu-latest
30+
defaults:
31+
run:
32+
working-directory: frontend
33+
steps:
34+
- uses: actions/checkout@v4
35+
- name: Restore cached build
36+
id: cache-build
37+
uses: actions/cache@v4
38+
with:
39+
path: frontend/build
40+
key: frontend-build-${{ hashFiles('frontend/**') }}
41+
restore-keys: |
42+
frontend-build-
43+
- name: Set up Node
44+
if: steps.cache-build.outputs.cache-hit != 'true'
45+
uses: actions/setup-node@v4
46+
with:
47+
node-version: 18
48+
- name: Install packages
49+
if: steps.cache-build.outputs.cache-hit != 'true'
50+
run: npm ci
51+
- name: Build dist
52+
if: steps.cache-build.outputs.cache-hit != 'true'
53+
run: npm run build
54+
- name: Upload dist
55+
uses: actions/upload-artifact@v4
56+
with:
57+
name: frontend-build
58+
path: frontend/build
59+
retention-days: 1
60+
61+
python-test:
62+
needs: [code-lint, frontend-build]
63+
runs-on: ${{ matrix.os }}
64+
strategy:
65+
matrix:
66+
os: [macos-latest, ubuntu-latest, windows-latest]
67+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
68+
steps:
69+
- uses: actions/checkout@v4
70+
- name: Set up Python ${{ matrix.python-version }}
71+
uses: astral-sh/setup-uv@v5
72+
with:
73+
python-version: ${{ matrix.python-version }}
74+
- name: Install dependencies
75+
run: uv sync --all-extras
76+
- name: Download frontend build
77+
uses: actions/download-artifact@v4
78+
with:
79+
name: frontend-build
80+
path: src/dstack/_internal/server/statics
81+
- name: Run pytest on POSIX
82+
if: matrix.os != 'windows-latest'
83+
# Skip Postgres tests on macos since macos runner doesn't have Docker.
84+
run: |
85+
RUNPOSTGRES=""
86+
if [ "${{ matrix.os }}" != "macos-latest" ]; then
87+
RUNPOSTGRES="--runpostgres"
88+
fi
89+
uv run pytest -n auto src/tests --runui $RUNPOSTGRES
90+
- name: Run pytest on Windows
91+
if: matrix.os == 'windows-latest'
92+
run: |
93+
uv run pytest -n auto src/tests --runui --runpostgres
94+
95+
runner-test:
96+
defaults:
97+
run:
98+
working-directory: runner
99+
runs-on: ${{ matrix.os }}
100+
strategy:
101+
matrix:
102+
os: [ubuntu-latest, macos-latest]
103+
steps:
104+
- uses: actions/checkout@v4
105+
- name: Set up Go
106+
uses: actions/setup-go@v5
107+
with:
108+
go-version-file: runner/go.mod
109+
cache-dependency-path: runner/go.sum
110+
- name: Check if go.mod and go.sum are up-to-date
111+
run: go mod tidy -diff
112+
- name: Run golangci-lint
113+
uses: golangci/golangci-lint-action@v6
114+
with:
115+
version: v1.62.0 # Should match .pre-commit-config.yaml
116+
args: --timeout=20m
117+
working-directory: runner
118+
- name: Test
119+
# Only run slow tests if requested by workflow call inputs.
120+
run: |
121+
SHORT="-short"
122+
if [[ "${{ inputs.go-integration-tests }}" == "true" ]]; then
123+
SHORT=""
124+
fi
125+
# Skip failing integration tests on macOS for release builds.
126+
# TODO: https://github.com/dstackai/dstack/issues/3005
127+
if [[ "${{ inputs.staging }}" == "false" && "${{ startsWith(matrix.os, 'macos') }}" == "true" ]]; then
128+
SHORT="-short"
129+
fi
130+
go version
131+
go fmt $(go list ./... | grep -v /vendor/)
132+
go vet $(go list ./... | grep -v /vendor/)
133+
go test $SHORT -race $(go list ./... | grep -v /vendor/)
134+
135+
runner-compile:
136+
needs: [runner-test]
137+
defaults:
138+
run:
139+
working-directory: runner
140+
env:
141+
REPO_NAME: github.com/dstackai/dstack
142+
strategy:
143+
matrix:
144+
include:
145+
- { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" }
146+
- { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" }
147+
runs-on: ${{ matrix.runs-on }}
148+
steps:
149+
- uses: actions/checkout@v4
150+
- name: Set up Go
151+
uses: actions/setup-go@v5
152+
with:
153+
go-version-file: runner/go.mod
154+
cache-dependency-path: runner/go.sum
155+
- name: build
156+
env:
157+
GOOS: ${{ matrix.goos }}
158+
GOARCH: ${{ matrix.goarch }}
159+
run: |
160+
CGO_ENABLED=0 go build -ldflags "-X 'main.Version=${{ inputs.version }}' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner
161+
CGO_ENABLED=1 go build -ldflags "-X 'main.Version=${{ inputs.version }}'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim
162+
- uses: actions/upload-artifact@v4
163+
with:
164+
name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
165+
path: |
166+
runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
167+
runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}
168+
retention-days: 1
169+
170+
gateway-build:
171+
runs-on: ubuntu-latest
172+
steps:
173+
- uses: actions/checkout@v4
174+
- name: Set up uv
175+
uses: astral-sh/setup-uv@v5
176+
with:
177+
python-version: 3.11
178+
- name: Build package
179+
working-directory: gateway
180+
run: |
181+
echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py
182+
# TODO: depend on a specific dstackai/dstack commit for staging builds?
183+
if [[ "${{ inputs.staging }}" == "false" ]]; then
184+
sed \
185+
-i.old \
186+
"s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ inputs.version }}|" \
187+
pyproject.toml
188+
diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1
189+
fi
190+
uv build
191+
- uses: actions/upload-artifact@v4
192+
with:
193+
name: dstack-gateway
194+
path: gateway/dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl
195+
retention-days: 1
196+
197+
python-build:
198+
needs: [code-lint, frontend-build]
199+
runs-on: ubuntu-latest
200+
steps:
201+
- uses: actions/checkout@v4
202+
- name: Set up uv
203+
uses: astral-sh/setup-uv@v5
204+
with:
205+
python-version: 3.11
206+
- name: Download frontend build
207+
uses: actions/download-artifact@v4
208+
with:
209+
name: frontend-build
210+
path: src/dstack/_internal/server/statics
211+
- name: Build dstack Python package
212+
# TODO: set __version__ to inputs.version regardless of inputs.staging,
213+
# so that staging builds are also tied to a specific runner and gateway version.
214+
# May require changing how dstack handles __version__.
215+
run: |
216+
if [[ "${{ inputs.staging }}" == "true" ]]; then
217+
VERSION=0.0.0
218+
IS_RELEASE=False
219+
else
220+
VERSION=${{ inputs.version }}
221+
IS_RELEASE=True
222+
fi
223+
BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ")
224+
BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ")
225+
echo "__version__ = \"$VERSION\"" > src/dstack/version.py
226+
echo "__is_release__ = $IS_RELEASE" >> src/dstack/version.py
227+
echo $BASE_IMAGE >> src/dstack/version.py
228+
echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py
229+
cp README.md src
230+
uv build
231+
- uses: actions/upload-artifact@v4
232+
with:
233+
name: python-build
234+
path: dist
235+
retention-days: 1
236+
237+
generate-json-schema:
238+
needs: [code-lint]
239+
runs-on: ubuntu-latest
240+
steps:
241+
- uses: actions/checkout@v4
242+
- uses: astral-sh/setup-uv@v5
243+
with:
244+
python-version: 3.11
245+
- name: Install dstack
246+
run: uv sync
247+
- name: Generate json schema
248+
run: |
249+
mkdir /tmp/json-schemas
250+
uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json
251+
uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > /tmp/json-schemas/profiles.json
252+
- uses: actions/upload-artifact@v4
253+
with:
254+
name: json-schemas
255+
path: /tmp/json-schemas
256+
retention-days: 1

‎.github/workflows/build-docs.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: Build Docs
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
release-tag:
7+
type: string
8+
required: false
9+
10+
jobs:
11+
build-docs:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: actions/checkout@v4
15+
- uses: astral-sh/setup-uv@v5
16+
with:
17+
python-version: 3.11
18+
- name: Install dstack
19+
run: |
20+
uv pip install examples/plugins/example_plugin_server
21+
if [ -n "${{ inputs.release-tag }}" ]; then
22+
uv pip install "dstack[server]==${{ inputs.release-tag }}"
23+
else
24+
uv pip install -e '.[server]'
25+
fi
26+
- name: Build
27+
run: |
28+
uv pip install pillow cairosvg
29+
sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev
30+
uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade
31+
uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
32+
uv run mkdocs build -s
33+
- uses: actions/upload-artifact@v4
34+
with:
35+
name: site
36+
path: site
37+
retention-days: 1

0 commit comments

Comments
 (0)