Merge branch 'master' into rekhoff/force-websocket-disconnect

Signed-off-by: rekhoff <r.ekhoff@clockworklabs.io>

Author: rekhoff

.env.sample

@@ -0,0 +1,6 @@
+# Minimal Dockerfile that just wraps pre-built binaries, so we can test the server inside docker
+FROM rust:1.90.0
+RUN mkdir -p /stdb/data
+COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli /usr/local/bin/
+COPY ./crates/standalone/config.toml /stdb/data/config.toml
+RUN ln -s /usr/local/bin/spacetimedb-cli /usr/local/bin/spacetime

.github/Dockerfile

@@ -0,0 +1,15 @@
+services:
+  node:
+    labels:
+      app: spacetimedb
+    build:
+      context: ../
+      dockerfile: .github/Dockerfile
+    ports:
+      - "3000:3000"
+      # Postgres
+      - "5432:5432"
+    entrypoint: spacetime start --pg-port 5432
+    privileged: true
+    environment:
+      RUST_BACKTRACE: 1

.github/docker-compose.yml

@@ -0,0 +1,67 @@
+name: Attach client binaries to release
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: "Release tag (e.g. v1.9.0)"
+        required: true
+
+jobs:
+  upload-assets:
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
+    permissions:
+      contents: write  # needed to modify releases
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download artifacts from private base URL
+        env:
+          RELEASE_TAG: ${{ github.event.inputs.release_tag }}
+          BASE_URL: ${{ secrets.ARTIFACT_BASE_URL }}
+        run: |
+          set -euo pipefail
+
+          FULL_URL="$BASE_URL/$RELEASE_TAG"
+
+          mkdir -p artifacts
+          cd artifacts
+
+          download() {
+            local filename="$1"
+            if ! wget -q "${FULL_URL}/${filename}" -O "${filename}"; then
+              echo "Failed to download ${filename}"
+              exit 1
+            fi
+          }
+
+          download "spacetime-aarch64-apple-darwin.tar.gz"
+          download "spacetime-aarch64-unknown-linux-gnu.tar.gz"
+          download "spacetime-x86_64-apple-darwin.tar.gz"
+          download "spacetime-x86_64-pc-windows-msvc.zip"
+          download "spacetime-x86_64-unknown-linux-gnu.tar.gz"
+          download "spacetimedb-update-aarch64-apple-darwin"
+          download "spacetimedb-update-aarch64-unknown-linux-gnu"
+          download "spacetimedb-update-x86_64-apple-darwin"
+          download "spacetimedb-update-x86_64-pc-windows-msvc.exe"
+          download "spacetimedb-update-x86_64-unknown-linux-gnu"
+
+      - name: Upload artifacts to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_TAG: ${{ github.event.inputs.release_tag }}
+        run: |
+          set -euo pipefail
+
+          cd artifacts
+
+          gh release upload "$RELEASE_TAG" ./* \
+            --repo "$GITHUB_REPOSITORY" \
+            --clobber
+

.github/workflows/attach-artifacts.yml

@@ -14,21 +14,28 @@ on:
 name: CI
 
 concurrency:
-  # When a PR number isn't available, the event won't be a `pull_request` event so it won't matter.
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  # Only cancel when the event is a pull_request
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || format('sha-{0}', github.sha) }}
+  cancel-in-progress: true
 
 jobs:
   docker_smoketests:
     name: Smoketests
     strategy:
       matrix:
+        runner: [spacetimedb-new-runner, windows-latest]
         include:
-          - { runner: spacetimedb-runner, smoketest_args: --docker }
-          - { runner: windows-latest, smoketest_args: --no-build-cli }
-        runner: [ spacetimedb-runner, windows-latest ]
+          - runner: spacetimedb-new-runner
+            smoketest_args: --docker
+            container:
+              image: localhost:5000/spacetimedb-ci:latest
+              options: --privileged
+          - runner: windows-latest
+            smoketest_args: --no-build-cli
+            container: null
     runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -47,24 +54,52 @@ jobs:
         with:
           ref: ${{ env.GIT_REF }}
       - uses: dsherret/rust-toolchain-file@v1
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          cache-on-failure: true
+          cache-all-crates: true
+          cache-workspace-crates: true
+
       - uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
+
+      # nodejs and pnpm are required for the typescript quickstart smoketest
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: true
+
       - name: Install psql (Windows)
         if: runner.os == 'Windows'
         run: choco install psql -y --no-progress
         shell: powershell
+      - name: Build crates
+        run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
+      - name: Start Docker daemon
+        if: runner.os == 'Linux'
+        run: /usr/local/bin/start-docker.sh
+
       - name: Build and start database (Linux)
         if: runner.os == 'Linux'
-        run: docker compose up -d
+        run: |
+          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
+          rm .dockerignore
+          docker compose -f .github/docker-compose.yml up -d
       - name: Build and start database (Windows)
         if: runner.os == 'Windows'
         run: |
           # Fail properly if any individual command fails
           $ErrorActionPreference = 'Stop'
           $PSNativeCommandUseErrorActionPreference = $true
 
-          cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
           Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
           cd modules
           # the sdk-manifests on windows-latest are messed up, so we need to update them
@@ -74,17 +109,23 @@ jobs:
         with: { python-version: '3.12' }
         if: runner.os == 'Windows'
       - name: Install python deps
-        run: python -m pip install psycopg2-binary xmltodict
+        run: python -m pip install -r smoketests/requirements.txt
       - name: Run smoketests
         # Note: clear_database and replication only work in private
-        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication
+        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
-        run: docker compose down
+        run: docker compose -f .github/docker-compose.yml down
 
   test:
     name: Test Suite
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -104,6 +145,13 @@ jobs:
           ref: ${{ env.GIT_REF }}
 
       - uses: dsherret/rust-toolchain-file@v1
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
 
       - uses: actions/setup-dotnet@v3
         with:
@@ -131,6 +179,10 @@ jobs:
         #Note: Unreal tests will be run separately
         run: cargo test --all -- --skip unreal
 
+        # The fallocate tests have been flakely when running in parallel
+      - name: Run fallocate tests
+        run: cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1
+
       - name: Check that the test outputs are up-to-date
         run: bash tools/check-diff.sh
 
@@ -145,14 +197,28 @@ jobs:
 
   lints:
     name: Lints
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
       - uses: dsherret/rust-toolchain-file@v1
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - uses: actions/setup-dotnet@v3
         with:
           global-json-file: global.json
@@ -184,13 +250,27 @@ jobs:
 
   wasm_bindings:
     name: Build and test wasm bindings
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - uses: actions/checkout@v3
 
       - uses: dsherret/rust-toolchain-file@v1
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - name: Run bindgen tests
         run: cargo test -p spacetimedb-codegen
 
@@ -209,7 +289,11 @@ jobs:
 
   publish_checks:
     name: Check that packages are publishable
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     permissions: read-all
     steps:
       - uses: actions/checkout@v3
@@ -219,11 +303,12 @@ jobs:
           venv/bin/pip3 install argparse toml
       - name: Run checks
         run: |
+          set -ueo pipefail
           FAILED=0
-          # This definition of ROOTS and invocation of find-publish-list.py is copied from publish-crates.sh
-          ROOTS=(bindings sdk cli standalone)
-          for crate in $(venv/bin/python3 tools/find-publish-list.py --recursive --quiet "${ROOTS[@]}"); do
-            if ! venv/bin/python3 tools/crate-publish-checks.py "crates/$crate"; then
+          ROOTS=(spacetimedb spacetimedb-sdk)
+          CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}")
+          for crate_dir in $CRATES; do
+            if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then
               FAILED=$(( $FAILED + 1 ))
             fi
           done
@@ -276,7 +361,12 @@ jobs:
     name: Unreal Engine Tests
     # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
     # a custom runner.
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    # Disable the tests because they are very flaky at the moment.
+    # TODO: Remove this line and re-enable the `if` line just below here.
+    if: false
+    # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
+    # if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
     container:
       image: ghcr.io/epicgames/unreal-engine:dev-5.6
       credentials:
@@ -353,13 +443,19 @@ jobs:
 
               cd "$GITHUB_WORKSPACE/sdks/unreal"
               cargo --version
-              cargo test
+              cargo test -- --test-threads=1
             '
 
   cli_docs:
     name: Check CLI docs
     permissions: read-all
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -403,12 +499,20 @@ jobs:
 
       - uses: dsherret/rust-toolchain-file@v1
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - name: Check for docs change
         run: |
           cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md
           pnpm format
           git status
-          if git diff --exit-code HEAD; then
+          if git diff --exit-code HEAD -- docs/docs/cli-reference.md; then
             echo "No docs changes detected"
           else
             echo "It looks like the CLI docs have changed:"