Merge remote-tracking branch 'origin/master' into kim/cloud/dev

Author: kim

.github/workflows/ci.yml

@@ -101,8 +101,15 @@ jobs:
 
       - name: Install psql (Windows)
         if: runner.os == 'Windows'
-        run: choco install psql -y --no-progress
-        shell: powershell
+        shell: pwsh
+        run: |
+          # Fail properly if any individual command fails
+          $ErrorActionPreference = 'Stop'
+          $PSNativeCommandUseErrorActionPreference = $true
+          choco install psql -y --no-progress
+          # Check for existence, since `choco` doesn't seem to fail the step if it fails to install..
+          # See https://github.com/clockworklabs/SpacetimeDB/pull/4399 for more background.
+          Get-Command psql
 
       - name: Update dotnet workloads
         if: runner.os == 'Windows'
@@ -171,132 +178,6 @@ jobs:
               exit 1
           }
 
-  smoketests-python:
-    needs: [lints]
-    name: Smoketests (Python Legacy) (${{matrix.name}})
-    strategy:
-      matrix:
-        include:
-          - name: Linux
-            runner: spacetimedb-new-runner-2
-            smoketest_args: --docker
-          - name: Windows
-            runner: windows-latest
-            smoketest_args: --no-build-cli
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 120
-    env:
-      CARGO_TARGET_DIR: ${{ github.workspace }}/target
-    steps:
-      - name: Find Git ref
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        shell: bash
-        run: |
-          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
-          if test -n "${PR_NUMBER}"; then
-            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
-          else
-            GIT_REF="${{ github.ref }}"
-          fi
-          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
-      - name: Checkout sources
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ env.GIT_REF }}
-      - uses: dsherret/rust-toolchain-file@v1
-      - name: Set default rust toolchain
-        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: ${{ github.workspace }}
-          shared-key: spacetimedb
-          cache-on-failure: false
-          cache-all-crates: true
-          cache-workspace-crates: true
-          prefix-key: v1
-
-      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
-      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
-      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
-      # `cargo build --manifest-path` (which apparently build different dependency trees).
-      # However, we've been unable to fix it so... /shrug
-      - name: Check v8 outputs
-        shell: bash
-        run: |
-          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
-          if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
-            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
-            cargo clean -p v8 || true
-            cargo build -p v8
-          fi
-
-      - uses: actions/setup-dotnet@v4
-        with:
-          global-json-file: global.json
-
-      - name: Override NuGet packages
-        shell: bash
-        run: |
-          dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime
-          dotnet pack -c Release crates/bindings-csharp/Runtime
-          cd sdks/csharp
-          ./tools~/write-nuget-config.sh ../..
-
-      # nodejs and pnpm are required for the typescript quickstart smoketest
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - uses: pnpm/action-setup@v4
-        with:
-          run_install: true
-
-      - name: Install psql (Windows)
-        if: runner.os == 'Windows'
-        run: choco install psql -y --no-progress
-        shell: powershell
-
-      - name: Build crates
-        run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
-
-      - name: Build and start database (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
-          rm .dockerignore
-          docker compose -f .github/docker-compose.yml up -d
-
-      - name: Build and start database (Windows)
-        if: runner.os == 'Windows'
-        run: |
-          # Fail properly if any individual command fails
-          $ErrorActionPreference = 'Stop'
-          $PSNativeCommandUseErrorActionPreference = $true
-
-          Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
-          cd modules
-          # the sdk-manifests on windows-latest are messed up, so we need to update them
-          dotnet workload config --update-mode manifests
-          dotnet workload update
-
-      - uses: actions/setup-python@v5
-        with: { python-version: "3.12" }
-        if: runner.os == 'Windows'
-
-      - name: Install python deps
-        run: python -m pip install -r smoketests/requirements.txt
-
-      - name: Run Python smoketests
-        # Note: clear_database and replication only work in private
-        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
-
-      - name: Stop containers (Linux)
-        if: always() && runner.os == 'Linux'
-        run: docker compose -f .github/docker-compose.yml down
-
   test:
     needs: [lints]
     name: Test Suite
@@ -1180,3 +1061,59 @@ jobs:
           fi
 
           echo "No Python smoketest changes detected."
+
+  smoketests_mod_rs_complete:
+    name: Check smoketests/mod.rs is complete
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+    steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Set default rust toolchain
+        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          cache-on-failure: false
+          cache-all-crates: true
+          cache-workspace-crates: true
+          prefix-key: v1
+
+      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
+      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
+      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
+      # `cargo build --manifest-path` (which apparently build different dependency trees).
+      # However, we've been unable to fix it so... /shrug
+      - name: Check v8 outputs
+        shell: bash
+        run: |
+          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
+          if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
+            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
+            cargo clean -p v8 || true
+            cargo build -p v8
+          fi
+
+      - name: Verify crates/smoketests/tests/smoketests/mod.rs lists all entries
+        run: |
+          cargo ci smoketests check-mod-list

.github/workflows/package.yml

@@ -4,12 +4,14 @@ on:
   push:
     tags:
       - '**'
-    branches:
-      - master
-      - release/*
+  workflow_dispatch:
+
+permissions:
+  contents: read
 
 jobs:
   build-cli:
+    if: ${{ !(startsWith(github.ref, 'refs/tags/') && matrix.target == 'x86_64-pc-windows-msvc') }}
     strategy:
       fail-fast: false
       matrix:
@@ -85,3 +87,89 @@ jobs:
           source_dir: build
           endpoint: https://nyc3.digitaloceanspaces.com
           destination_dir: ${{ steps.extract_branch.outputs.branch }}
+
+  build-cli-windows-signed:
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    name: Build and sign CLI for x86_64 Windows
+    runs-on: [self-hosted, windows, signing]
+    environment: codesign
+    concurrency:
+      group: codesign-${{ github.ref }}
+      cancel-in-progress: false
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Rust
+        uses: dsherret/rust-toolchain-file@v1
+
+      - name: Install rust target
+        run: rustup target add x86_64-pc-windows-msvc
+
+      - name: Compile
+        run: |
+          cargo build --release --target x86_64-pc-windows-msvc -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
+
+      - name: Write certificate file
+        shell: powershell
+        env:
+          DIGICERT_CERT_B64: ${{ secrets.DIGICERT_CERT_B64 }}
+        run: |
+          [IO.File]::WriteAllBytes("digicert.crt", [Convert]::FromBase64String($env:DIGICERT_CERT_B64))
+
+      - name: Sign binaries
+        shell: powershell
+        env:
+          DIGICERT_KEYPAIR_ALIAS: ${{ secrets.DIGICERT_KEYPAIR_ALIAS }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $targetDir = Join-Path $env:GITHUB_WORKSPACE 'target\x86_64-pc-windows-msvc\release'
+          $certFile = Join-Path $env:GITHUB_WORKSPACE 'digicert.crt'
+
+          $signtool = Get-Command signtool.exe -ErrorAction Stop
+
+          $files = @(
+            (Join-Path $targetDir 'spacetimedb-update.exe'),
+            (Join-Path $targetDir 'spacetimedb-cli.exe'),
+            (Join-Path $targetDir 'spacetimedb-standalone.exe')
+          )
+
+          foreach ($file in $files) {
+            & $signtool.Path sign /csp "DigiCert Signing Manager KSP" /kc $env:DIGICERT_KEYPAIR_ALIAS /f $certFile /tr http://timestamp.digicert.com /td SHA256 /fd SHA256 $file
+            & $signtool.Path verify /v /pa $file
+          }
+
+      - name: Package (windows)
+        shell: powershell
+        run: |
+          $ErrorActionPreference = 'Stop'
+          New-Item -ItemType Directory -Force -Path build | Out-Null
+          $releaseDir = Join-Path $env:GITHUB_WORKSPACE 'target\x86_64-pc-windows-msvc\release'
+
+          Copy-Item (Join-Path $releaseDir 'spacetimedb-update.exe') (Join-Path $env:GITHUB_WORKSPACE 'build\spacetimedb-update-x86_64-pc-windows-msvc.exe')
+          Compress-Archive -Force -Path @(
+            (Join-Path $releaseDir 'spacetimedb-cli.exe'),
+            (Join-Path $releaseDir 'spacetimedb-standalone.exe')
+          ) -DestinationPath (Join-Path $env:GITHUB_WORKSPACE 'build\spacetime-x86_64-pc-windows-msvc.zip')
+
+      - name: Extract branch name
+        shell: powershell
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $branch = $env:GITHUB_HEAD_REF
+          if ([string]::IsNullOrEmpty($branch)) {
+            $branch = $env:GITHUB_REF -replace '^refs/heads/', ''
+          }
+          "branch=$branch" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+        id: extract_branch
+
+      - name: Upload to DO Spaces
+        uses: shallwefootball/s3-upload-action@master
+        with:
+          aws_key_id: ${{ secrets.AWS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+          aws_bucket: ${{ vars.AWS_BUCKET }}
+          source_dir: build
+          endpoint: https://nyc3.digitaloceanspaces.com
+          destination_dir: ${{ steps.extract_branch.outputs.branch }}