build.yml

name: Build

on:
  push:
    branches:
      - "master"
    paths-ignore:
      - "docs/**"
      - "mkdocs.yml"
  pull_request:
    branches:
      - "master"
  workflow_dispatch:
    inputs:
      intergation-tests:
        type: boolean
        required: true
        default: false

env:
  BUILD_INCREMENT: 150
  PIP_DISABLE_PIP_VERSION_CHECK: on
  PIP_DEFAULT_TIMEOUT: 10
  PIP_PROGRESS_BAR: off

jobs:
  python-lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up uv
        uses: astral-sh/setup-uv@v5
        with:
          python-version: 3.11
      - run: uv tool install pre-commit
      - run: pre-commit run -a --show-diff-on-failure

  frontend-build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: frontend
    steps:
      - uses: actions/checkout@v4
      - name: Restore cached build
        id: cache-build
        uses: actions/cache@v4
        with:
          path: frontend/build
          key: frontend-build-${{ hashFiles('frontend/**') }}
          restore-keys: |
            frontend-build-
      - name: Set up Node
        if: steps.cache-build.outputs.cache-hit != 'true'
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Install packages
        if: steps.cache-build.outputs.cache-hit != 'true'
        run: npm ci
      - name: Build dist
        if: steps.cache-build.outputs.cache-hit != 'true'
        run: npm run build
      - name: Upload dist
        uses: actions/upload-artifact@v4
        with:
          name: frontend-build
          path: frontend/build

  python-test:
    needs: [python-lint, frontend-build]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [macos-latest, ubuntu-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: astral-sh/setup-uv@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: uv sync --all-extras
      - name: Download frontend build
        uses: actions/download-artifact@v4
        with:
          name: frontend-build
          path: src/dstack/_internal/server/statics
      - name: Run pytest on POSIX
        if: matrix.os != 'windows-latest'
        # Skip Postgres tests on macos since macos runner doesn't have Docker.
        run: |
          RUNPOSTGRES=""
          if [ "${{ matrix.os }}" != "macos-latest" ]; then
            RUNPOSTGRES="--runpostgres"
          fi
          uv run pytest -n auto src/tests --runui $RUNPOSTGRES
      - name: Run pytest on Windows
        if: matrix.os == 'windows-latest'
        run: |
          uv run pytest -n auto src/tests --runui --runpostgres

  update-get-dstack:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    needs: [python-test]
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    steps:
      - name: Set up uv
        uses: astral-sh/setup-uv@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: uv tool install awscli
      - run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          echo $VERSION | aws s3 cp - s3://get-dstack/stgn-cli/latest-version --acl public-read

  runner-test:
    defaults:
      run:
        working-directory: runner
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: runner/go.mod
          cache-dependency-path: runner/go.sum
      - name: Check if go.mod and go.sum are up-to-date
        run: go mod tidy -diff
      - name: Run golangci-lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: v1.62.0
          args: --timeout=20m
          working-directory: runner
      - name: Test
        # Do not run slow integration tests automatically.
        # Slow tests can be run manually via workflow_dispatch when required.
        run: |
          SHORT="-short"
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            if [[ "${{ github.event.inputs.intergation-tests }}" == "true" ]]; then
              SHORT=""
            fi
          fi
          go version
          go fmt $(go list ./... | grep -v /vendor/)
          go vet $(go list ./... | grep -v /vendor/)
          go test $SHORT -race $(go list ./... | grep -v /vendor/)

  runner-compile:
    needs: [runner-test]
    defaults:
      run:
        working-directory: runner
    env:
      REPO_NAME: github.com/dstackai/dstack
    strategy:
      matrix:
        include:
          - { goos: "linux", goarch: "amd64" }
          - { goos: "linux", goarch: "arm64" }
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: runner/go.mod
          cache-dependency-path: runner/go.sum
      - name: build
        env:
          GOOS: ${{ matrix.goos }}
          GOARCH: ${{ matrix.goarch }}
          CGO_ENABLED: 0
        run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner
          go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim
          echo $VERSION
      - uses: actions/upload-artifact@v4
        with:
          name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
          path: |
            runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
            runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}
          retention-days: 1

  runner-upload:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    needs: [runner-compile]
    runs-on: ubuntu-latest
    steps:
      - name: Set up uv
        uses: astral-sh/setup-uv@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: uv tool install awscli
      - name: Download Runner
        uses: actions/download-artifact@v4
        with:
          pattern: dstack-runner-*
          merge-multiple: true
          path: runner
      - name: Upload to S3
        working-directory: runner
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          aws s3 cp . "s3://dstack-runner-downloads-stgn/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read
          aws s3 cp . "s3://dstack-runner-downloads-stgn/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read

  generate-json-schema:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    needs: [python-test]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: uv tool install awscli
      - name: Install dstack
        run: uv sync
      - name: Generate json schema
        run: |
          uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json(indent=2))" > configuration.json
          uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json(indent=2))" > profiles.json
      - name: Upload json schema to S3
        run: |
          VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
          aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/configuration.json" --acl public-read
          aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/latest/schemas/configuration.json" --acl public-read
          aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/profiles.json" --acl public-read
          aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/latest/schemas/profiles.json" --acl public-read

  gateway-build:
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: gateway
    steps:
      - uses: actions/checkout@v4
      - name: Set up uv
        uses: astral-sh/setup-uv@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: uv tool install awscli
      - name: Compute version
        run: echo VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) > $GITHUB_ENV
      - name: Build package
        run: |
          echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py
          uv build
      - name: Upload to S3
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl
          aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/stgn/$WHEEL"
          echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/stgn/latest-version"

  docs-build:
    # Skip for PRs from forks since mkdocs-material-insiders is not available in forks
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          python-version: 3.11
      - name: Install dstack
        run: |
          uv pip install examples/plugins/example_plugin_server
          uv pip install -e '.[server]'
      # Move these deps into an extra and install that way
      - name: Build
        run: |
          uv pip install pillow cairosvg
          sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev
          uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade
          uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
          uv run mkdocs build -s