Migrate to uv for development and CI (#2455)

* Migrate to pyproject.toml

* Include server extras in backends extras

* Use uv for release workflow

* Mention uv sync --all-extras --active

* Remove --frozen

* Use uv to publish to pypi

* Replace pip with uv in actions

* Require httpbin to be compatible with Werkzeug 3

---------

Co-authored-by: Victor Skvortsov <vds003@gmail.com>

Author: colinjc

.github/workflows/build.yml

@@ -3,13 +3,13 @@ name: Build
 on:
   push:
     branches:
-      - 'master'
+      - "master"
     paths-ignore:
-      - 'docs/**'
-      - 'mkdocs.yml'
+      - "docs/**"
+      - "mkdocs.yml"
   pull_request:
     branches:
-      - 'master'
+      - "master"
   workflow_dispatch:
     inputs:
       intergation-tests:
@@ -56,20 +56,20 @@ jobs:
           path: frontend/build
 
   python-test:
-    needs: [ python-lint, frontend-build ]
+    needs: [python-lint, frontend-build]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ macos-latest, ubuntu-latest, windows-latest ]
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        os: [macos-latest, ubuntu-latest, windows-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: astral-sh/setup-uv@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        run: pip install -U '.[all]' -r requirements_dev.txt
+        run: uv sync --all-extras
       - name: Download frontend build
         uses: actions/download-artifact@v4
         with:
@@ -83,22 +83,26 @@ jobs:
           if [ "${{ matrix.os }}" != "macos-latest" ]; then
             RUNPOSTGRES="--runpostgres"
           fi
-          pytest src/tests --runui $RUNPOSTGRES
+          uv run pytest src/tests --runui $RUNPOSTGRES
       - name: Run pytest on Windows
         if: matrix.os == 'windows-latest'
         run: |
-          pytest src/tests --runui --runpostgres
+          uv run pytest src/tests --runui --runpostgres
 
   update-get-dstack:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    needs: [ python-test ]
+    needs: [python-test]
     runs-on: ubuntu-latest
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     steps:
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: 3.11
       - name: Install AWS
-        run: pip install awscli
+        run: uv tool install awscli
       - run: |
           VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
           echo $VERSION | aws s3 cp - s3://get-dstack/stgn-cli/latest-version --acl public-read
@@ -181,8 +185,12 @@ jobs:
     needs: [runner-compile]
     runs-on: ubuntu-latest
     steps:
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: 3.11
       - name: Install AWS
-        run: pip install awscli
+        run: uv tool install awscli
       - name: Download Runner
         uses: actions/download-artifact@v4
         with:
@@ -201,24 +209,24 @@ jobs:
 
   generate-json-schema:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    needs: [ python-test ]
+    needs: [python-test]
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v5
         with:
           python-version: 3.11
       - name: Install AWS
-        run: pip install awscli
+        run: uv tool install awscli
       - name: Install dstack
-        run: pip install .
+        run: uv sync
       - name: Generate json schema
         run: |
-          python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json(indent=2))" > configuration.json
-          python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json(indent=2))" > profiles.json
+          uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json(indent=2))" > configuration.json
+          uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json(indent=2))" > profiles.json
       - name: Upload json schema to S3
         run: |
           VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))
@@ -235,20 +243,18 @@ jobs:
         working-directory: gateway
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
         with:
           python-version: 3.11
       - name: Install AWS
-        run: pip install awscli
-      - name: Install dependencies
-        run: pip install wheel build
+        run: uv tool install awscli
       - name: Compute version
         run: echo VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) > $GITHUB_ENV
       - name: Build package
         run: |
           echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py
-          python -m build .
+          uv build
       - name: Upload to S3
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -257,23 +263,24 @@ jobs:
           WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl
           aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/stgn/$WHEEL"
           echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/stgn/latest-version"
-  
+
   docs-build:
     # Skip for PRs from forks since mkdocs-material-insiders is not available in forks
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v5
         with:
           python-version: 3.11
       - name: Install dstack
         run: |
-          pip install -e .[server]
+          uv sync --extra server
+      # Move these deps into an extra and install that way
       - name: Build
         run: |
-          pip install pillow cairosvg
+          uv pip install pillow cairosvg
           sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev
-          pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade
-          pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
-          mkdocs build -s
+          uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade
+          uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
+          uv mkdocs build -s

.github/workflows/docker.yml

@@ -34,7 +34,7 @@ on:
 
 env:
   PACKER_VERSION: "1.9.2"
-  BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }}  # staging ? prefix : ''
+  BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }} # staging ? prefix : ''
 
 jobs:
   build-docker:
@@ -45,23 +45,23 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
-     - name: Checkout repository
-       uses: actions/checkout@v4
-     - name: Set up Docker Buildx
-       uses: docker/setup-buildx-action@v3
-     - name: Login to DockerHub
-       uses: docker/login-action@v3
-       with:
-         username: ${{ secrets.DOCKERHUB_USERNAME }}
-         password: ${{ secrets.DOCKERHUB_TOKEN }}
-     - name: Set up QEMU
-       uses: docker/setup-qemu-action@v3
-     - name: Build and upload to DockerHub
-       run: |
-         docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1 -f base/Dockerfile .
-         docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --build-arg VERSION=${{ inputs.image_version }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1-devel -f base/devel.Dockerfile .
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Build and upload to DockerHub
+        run: |
+          docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1 -f base/Dockerfile .
+          docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --build-arg VERSION=${{ inputs.image_version }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1-devel -f base/devel.Dockerfile .
 
   build-aws-images:
     needs: build-docker
@@ -75,7 +75,7 @@ jobs:
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     strategy:
       matrix:
-        variant: [ "", "-cuda" ]
+        variant: ["", "-cuda"]
     steps:
       - uses: actions/checkout@v4
       - name: Download packer
@@ -87,7 +87,7 @@ jobs:
         run: |
           ./packer build -var-file=versions.json $PROD_VARS -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX aws-image${{ matrix.variant }}.json
         env:
-          PROD_VARS: ${{ !inputs.staging && '-var-file=aws-vars-prod.json' || '' }}  # production ? var-file : ''
+          PROD_VARS: ${{ !inputs.staging && '-var-file=aws-vars-prod.json' || '' }} # production ? var-file : ''
 
   build-azure-images:
     needs: build-docker
@@ -104,7 +104,7 @@ jobs:
       VERSION: ${{ github.run_number }}
     strategy:
       matrix:
-        variant: [ "", "-cuda", "-grid" ]
+        variant: ["", "-cuda", "-grid"]
     steps:
       - uses: actions/checkout@v4
       - uses: Azure/login@v2
@@ -135,17 +135,17 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        variant: [ "", "-cuda" ]
+        variant: ["", "-cuda"]
     permissions:
-      contents: 'read'
-      id-token: 'write'
+      contents: "read"
+      id-token: "write"
     steps:
       - uses: actions/checkout@v4
       - name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
         with:
-          workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider'
-          service_account: 'github-actions@dstack.iam.gserviceaccount.com'
+          workload_identity_provider: "projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider"
+          service_account: "github-actions@dstack.iam.gserviceaccount.com"
           create_credentials_file: true
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v2
@@ -175,7 +175,7 @@ jobs:
       OCI_REGION: eu-frankfurt-1
     strategy:
       matrix:
-        variant: [ "", "-cuda" ]
+        variant: ["", "-cuda"]
     steps:
       - uses: actions/checkout@v4
       - name: Setup OCI config
@@ -211,23 +211,23 @@ jobs:
             -var oci_subnet_ocid=$OCI_SUBNET \
             -var oci_availability_domain=$OCI_AVAILABILITY_DOMAIN \
             oci-image${{ matrix.variant }}.json
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v5
         with:
-          python-version: '3.12'
+          python-version: "3.12"
       - name: Install dependencies for publishing
         run: |
-          pip install .[oci]
+          uv sync --extra oci
       - name: Copy image to target regions
         if: ${{ !inputs.staging }}
         run: |
-          python scripts/oci_image_tools.py copy \
+          uv run scripts/oci_image_tools.py copy \
             --image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
             --from $OCI_REGION \
             --compartment $OCI_COMPARTMENT
       - name: Publish image in OCI Marketplace
         if: ${{ !inputs.staging }}
         run: |
-          python scripts/oci_image_tools.py publish \
+          uv run scripts/oci_image_tools.py publish \
             --image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \
             --compartment $OCI_COMPARTMENT \
             --version ${{ inputs.image_version }} \

.github/workflows/docs.yaml

@@ -11,23 +11,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v5
         with:
           python-version: 3.11
       - name: Install dstack
         run: |
           if [ -n "${{ inputs.release_tag }}" ]; then
-            pip install "dstack[server]==${{ inputs.release_tag }}"
+            uv pip install "dstack[server]==${{ inputs.release_tag }}"
           else
-            pip install -e .[server]
+            uv sync --extra server
           fi
       - name: Build
         run: |
-          pip install pillow cairosvg
+          uv pip install pillow cairosvg
           sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev
-          pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade
-          pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
-          mkdocs build -s
+          uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade
+          uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
+          uv mkdocs build -s
       - name: Deploy
         uses: JamesIves/github-pages-deploy-action@v4.6.4
         with: