Skip to content

Commit de58bf3

Browse files
jvstmejvstme
committed
CI refactoring WIP
1 parent 3e54439 commit de58bf3

5 files changed

Lines changed: 366 additions & 456 deletions

File tree

‎.github/workflows/build-artifacts.yml‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
name: Build Artifacts
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
version:
7+
type: string
8+
required: true
9+
staging:
10+
type: boolean
11+
required: true
12+
default: true
13+
go-integration-tests:
14+
type: boolean
15+
required: true
16+
default: true
17+
18+
env:
19+
BUILD_INCREMENT: 150
20+
PIP_DISABLE_PIP_VERSION_CHECK: on
21+
PIP_DEFAULT_TIMEOUT: 10
22+
PIP_PROGRESS_BAR: off
23+
24+
jobs:
25+
code-lint:
26+
runs-on: ubuntu-latest
27+
steps:
28+
- uses: actions/checkout@v4
29+
- name: Set up uv
30+
uses: astral-sh/setup-uv@v5
31+
with:
32+
python-version: 3.11
33+
- run: uv tool install pre-commit
34+
- run: pre-commit run -a --show-diff-on-failure
35+
36+
frontend-build:
37+
runs-on: ubuntu-latest
38+
defaults:
39+
run:
40+
working-directory: frontend
41+
steps:
42+
- uses: actions/checkout@v4
43+
- name: Restore cached build
44+
id: cache-build
45+
uses: actions/cache@v4
46+
with:
47+
path: frontend/build
48+
key: frontend-build-${{ hashFiles('frontend/**') }}
49+
restore-keys: |
50+
frontend-build-
51+
- name: Set up Node
52+
if: steps.cache-build.outputs.cache-hit != 'true'
53+
uses: actions/setup-node@v4
54+
with:
55+
node-version: 18
56+
- name: Install packages
57+
if: steps.cache-build.outputs.cache-hit != 'true'
58+
run: npm ci
59+
- name: Build dist
60+
if: steps.cache-build.outputs.cache-hit != 'true'
61+
run: npm run build
62+
- name: Upload dist
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: frontend-build
66+
path: frontend/build
67+
68+
python-test:
69+
needs: [code-lint, frontend-build]
70+
runs-on: ${{ matrix.os }}
71+
strategy:
72+
matrix:
73+
os: [macos-latest, ubuntu-latest, windows-latest]
74+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
75+
steps:
76+
- uses: actions/checkout@v4
77+
- name: Set up Python ${{ matrix.python-version }}
78+
uses: astral-sh/setup-uv@v5
79+
with:
80+
python-version: ${{ matrix.python-version }}
81+
- name: Install dependencies
82+
run: uv sync --all-extras
83+
- name: Download frontend build
84+
uses: actions/download-artifact@v4
85+
with:
86+
name: frontend-build
87+
path: src/dstack/_internal/server/statics
88+
- name: Run pytest on POSIX
89+
if: matrix.os != 'windows-latest'
90+
# Skip Postgres tests on macos since macos runner doesn't have Docker.
91+
run: |
92+
RUNPOSTGRES=""
93+
if [ "${{ matrix.os }}" != "macos-latest" ]; then
94+
RUNPOSTGRES="--runpostgres"
95+
fi
96+
uv run pytest -n auto src/tests --runui $RUNPOSTGRES
97+
- name: Run pytest on Windows
98+
if: matrix.os == 'windows-latest'
99+
run: |
100+
uv run pytest -n auto src/tests --runui --runpostgres
101+
102+
runner-test:
103+
defaults:
104+
run:
105+
working-directory: runner
106+
runs-on: ${{ matrix.os }}
107+
strategy:
108+
matrix:
109+
os: [ubuntu-latest, macos-latest]
110+
steps:
111+
- uses: actions/checkout@v4
112+
- name: Set up Go
113+
uses: actions/setup-go@v5
114+
with:
115+
go-version-file: runner/go.mod
116+
cache-dependency-path: runner/go.sum
117+
- name: Check if go.mod and go.sum are up-to-date
118+
run: go mod tidy -diff
119+
- name: Run golangci-lint
120+
uses: golangci/golangci-lint-action@v6
121+
with:
122+
version: v1.62.0 # Should match .pre-commit-config.yaml
123+
args: --timeout=20m
124+
working-directory: runner
125+
- name: Test
126+
# Only run slow tests if requested by workflow call inputs.
127+
run: |
128+
SHORT="-short"
129+
if [[ "${{ github.event_name }}" == "workflow_call" ]]; then
130+
if [[ "${{ github.event.inputs.go-integration-tests }}" == "true" ]]; then
131+
SHORT=""
132+
fi
133+
fi
134+
go version
135+
go fmt $(go list ./... | grep -v /vendor/)
136+
go vet $(go list ./... | grep -v /vendor/)
137+
go test $SHORT -race $(go list ./... | grep -v /vendor/)
138+
139+
runner-compile:
140+
needs: [runner-test]
141+
defaults:
142+
run:
143+
working-directory: runner
144+
env:
145+
REPO_NAME: github.com/dstackai/dstack
146+
strategy:
147+
matrix:
148+
include:
149+
- { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" }
150+
- { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" }
151+
runs-on: ${{ matrix.runs-on }}
152+
steps:
153+
- uses: actions/checkout@v4
154+
- name: Set up Go
155+
uses: actions/setup-go@v5
156+
with:
157+
go-version-file: runner/go.mod
158+
cache-dependency-path: runner/go.sum
159+
- name: build
160+
env:
161+
GOOS: ${{ matrix.goos }}
162+
GOARCH: ${{ matrix.goarch }}
163+
run: |
164+
CGO_ENABLED=0 go build -ldflags "-X 'main.Version=${{ inputs.version }}' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner
165+
CGO_ENABLED=1 go build -ldflags "-X 'main.Version=${{ inputs.version }}'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim
166+
- uses: actions/upload-artifact@v4
167+
with:
168+
name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
169+
path: |
170+
runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
171+
runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}
172+
retention-days: 1
173+
174+
gateway-build:
175+
runs-on: ubuntu-latest
176+
defaults:
177+
run:
178+
working-directory: gateway
179+
steps:
180+
- uses: actions/checkout@v4
181+
- name: Set up uv
182+
uses: astral-sh/setup-uv@v5
183+
with:
184+
python-version: 3.11
185+
- name: Build package
186+
run: |
187+
echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py
188+
# TODO: depend on a specific dstackai/dstack commit for staging builds?
189+
if [[ "${{ inputs.staging }}" == "false" ]]; then
190+
sed \
191+
-i.old \
192+
"s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ inputs.version }}|" \
193+
pyproject.toml
194+
diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1
195+
fi
196+
uv build
197+
- uses: actions/upload-artifact@v4
198+
with:
199+
name: dstack-gateway
200+
path: dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl
201+
retention-days: 1
202+
203+
python-build:
204+
needs: [code-lint, frontend-build]
205+
runs-on: ubuntu-latest
206+
steps:
207+
- uses: actions/checkout@v4
208+
- name: Set up uv
209+
uses: astral-sh/setup-uv@v5
210+
with:
211+
python-version: 3.11
212+
- name: Download frontend build
213+
uses: actions/download-artifact@v4
214+
with:
215+
name: frontend-build
216+
path: src/dstack/_internal/server/statics
217+
- name: Build dstack Python package
218+
# TODO: set __version__ to inputs.version regardless of inputs.staging,
219+
# so that staging builds are also tied to a specific runner and gateway version.
220+
run: |
221+
if [[ "${{ inputs.staging }}" == "true" ]]; then
222+
VERSION=0.0.0
223+
IS_RELEASE=False
224+
else
225+
VERSION=${{ inputs.version }}
226+
IS_RELEASE=True
227+
fi
228+
BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ")
229+
BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ")
230+
echo "__version__ = \"$VERSION\"" > src/dstack/version.py
231+
echo "__is_release__ = $IS_RELEASE" >> src/dstack/version.py
232+
echo $BASE_IMAGE >> src/dstack/version.py
233+
echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py
234+
cp README.md src
235+
uv build
236+
- uses: actions/upload-artifact@v4
237+
with:
238+
name: python-build
239+
path: dist
240+
241+
generate-json-schema:
242+
needs: [code-lint]
243+
runs-on: ubuntu-latest
244+
steps:
245+
- uses: actions/checkout@v4
246+
- uses: astral-sh/setup-uv@v5
247+
with:
248+
python-version: 3.11
249+
- name: Install dstack
250+
run: uv sync
251+
- name: Generate json schema
252+
run: |
253+
mkdir /tmp/json-schemas
254+
uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json
255+
uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > /tmp/json-schemas/profiles.json
256+
- uses: actions/upload-artifact@v4
257+
with:
258+
name: json-schemas
259+
path: /tmp/json-schemas

0 commit comments

Comments
 (0)