Merge branch 'master' into bot/fix-listen-addr-help

Author: cloutiertyler

.cargo/config.toml

@@ -7,6 +7,7 @@ llm = "run --package xtask-llm-benchmark --bin llm_benchmark --"
 ci = "run -p ci --"
 smoketest = "ci smoketests --"
 smoketests = "smoketest"
+lint = "ci lint --"
 
 [target.x86_64-pc-windows-msvc]
 # Use a different linker. Otherwise, the build fails with some obscure linker error that

.github/CODEOWNERS

@@ -1,16 +1,17 @@
 /crates/core/src/db/datastore/traits.rs @cloutiertyler
 /rust-toolchain.toml @cloutiertyler
-/.github/CODEOWNERS @cloutiertyler
 LICENSE @cloutiertyler
 LICENSE.txt @cloutiertyler
 /licenses/ @cloutiertyler
 /crates/client-api-messages/src/websocket.rs @centril @gefjon
 
 /crates/cli/src/ @bfops @cloutiertyler @jdetter
 /tools/ci/ @bfops @cloutiertyler @jdetter
-/tools/upgrade-version/ @bfops @jdetter
-/tools/license-check/ @bfops @jdetter
-/.github/ @bfops @jdetter
+/tools/upgrade-version/ @bfops @jdetter @cloutiertyler
+/tools/license-check/ @bfops @jdetter @cloutiertyler
+/.github/ @bfops @jdetter @cloutiertyler
 
 /crates/sdk/examples/quickstart-chat/ @gefjon
 /modules/quickstart-chat/ @gefjon
+
+/.github/CODEOWNERS @cloutiertyler

.github/workflows/ci.yml

@@ -17,6 +17,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || format('sha-{0}', github.sha) }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  pull-requests: read
+
 jobs:
   smoketests:
     needs: [lints]
@@ -27,7 +31,7 @@ jobs:
           - name: Linux
             runner: spacetimedb-new-runner-2
           - name: Windows
-            runner: windows-latest
+            runner: spacetimedb-windows-runner
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 120
     env:
@@ -171,13 +175,6 @@ jobs:
           }
           cargo ci smoketests -- --test-threads=1          
 
-      - name: Check for changes
-        run: |
-          tools/check-diff.sh crates/smoketests || {
-              echo 'Error: There is a diff in the smoketests directory.'
-              exit 1
-          }
-
   test:
     needs: [lints]
     name: Test Suite
@@ -262,10 +259,6 @@ jobs:
 
           wasm-bindgen --version
 
-      - name: Build typescript module sdk
-        working-directory: crates/bindings-typescript
-        run: pnpm build
-
         # Source emsdk environment to make emcc (Emscripten compiler) available in PATH.
       - name: Run tests
         run: |
@@ -299,6 +292,15 @@ jobs:
         with:
           global-json-file: global.json
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: true
+
       - name: Run ci lint
         run: cargo ci lint
 
@@ -336,24 +338,8 @@ jobs:
       - uses: dsherret/rust-toolchain-file@v1
       - name: Set default rust toolchain
         run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
-      - name: Set up Python env
-        run: |
-          test -d venv || python3 -m venv venv
-          venv/bin/pip3 install argparse toml
       - name: Run checks
-        run: |
-          set -ueo pipefail
-          FAILED=0
-          ROOTS=(spacetimedb spacetimedb-sdk)
-          CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}")
-          for crate_dir in $CRATES; do
-            if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then
-              FAILED=$(( $FAILED + 1 ))
-            fi
-          done
-          if [ $FAILED -gt 0 ]; then
-            exit 1
-          fi
+        run: cargo ci publish-checks
 
   update:
     name: Test spacetimedb-update flow (${{ matrix.target }})
@@ -383,28 +369,10 @@ jobs:
         shell: bash
         run: sudo apt install -y libssl-dev
 
-      - name: Build spacetimedb-update
-        run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update
-        if: runner.os == 'Windows'
-
-      - name: Run self-install
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        shell: bash
-        run: |
-          ROOT_DIR="$(mktemp -d)"
-          # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
-          # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
-          # happens very frequently on the `macos-runner`, but we haven't seen it on any others).
-          cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
-          "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help
-        if: runner.os == 'Windows'
-
       - name: Test spacetimedb-update
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth
-        if: runner.os != 'Windows'
 
   unreal_engine_tests:
     name: Unreal Engine Tests
@@ -919,15 +887,16 @@ jobs:
             const targetRepo = process.env.TARGET_REPO;
             // Use the ref for pull requests because the head sha is brittle (github does some extra dance where it merges in master).
             const publicRef = (context.eventName === 'pull_request') ? context.payload.pull_request.head.ref : context.sha;
+            const publicPrNumber = context.payload.pull_request?.number ?? context.payload.inputs?.pr_number;
             const preDispatch = new Date().toISOString();
-  
+
             // Dispatch the workflow in the target repository
             await github.rest.actions.createWorkflowDispatch({
               owner: targetOwner,
               repo: targetRepo,
               workflow_id: workflowId,
               ref: targetRef,
-              inputs: { public_ref: publicRef }
+              inputs: { public_ref: publicRef, public_pr_number: String(publicPrNumber) }
             });
   
             const sleep = (ms) => new Promise(r => setTimeout(r, ms));
@@ -1056,36 +1025,6 @@ jobs:
       - name: Check global.json policy
         run: cargo ci global-json-policy
 
-  warn-python-smoketests:
-    name: Check for Python smoketest edits
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-    permissions:
-      contents: read
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Fail if Python smoketests were modified
-        run: |
-          MERGE_BASE="$(git merge-base origin/${{ github.base_ref }} HEAD)"
-          PYTHON_SMOKETEST_CHANGES="$(git diff --name-only "$MERGE_BASE" HEAD -- 'smoketests/**.py')"
-
-          if [ -n "$PYTHON_SMOKETEST_CHANGES" ]; then
-            echo "::error::This PR modifies legacy Python smoketests. Please add new tests to the Rust smoketests in crates/smoketests/ instead."
-            echo ""
-            echo "Changed files:"
-            echo "$PYTHON_SMOKETEST_CHANGES"
-            echo ""
-            echo "The Python smoketests are being replaced by Rust smoketests."
-            echo "See crates/smoketests/DEVELOP.md for instructions on adding Rust smoketests."
-            exit 1
-          fi
-
-          echo "No Python smoketest changes detected."
-
   smoketests_mod_rs_complete:
     name: Check smoketests/mod.rs is complete
     runs-on: ubuntu-latest
@@ -1141,3 +1080,151 @@ jobs:
       - name: Verify crates/smoketests/tests/smoketests/mod.rs lists all entries
         run: |
           cargo ci smoketests check-mod-list
+
+  docs-build:
+    name: Docs build
+    runs-on: spacetimedb-new-runner-2
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '22'
+
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: true
+
+      - name: Get pnpm store directory
+        working-directory: sdks/typescript
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Set default rust toolchain
+        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
+
+      - name: Docusaurus build
+        run: cargo ci docs
+
+  typescript-test:
+    name: TypeScript - Tests
+    runs-on: spacetimedb-new-runner-2
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: true
+
+      - name: Get pnpm store directory
+        shell: bash
+        working-directory: crates/bindings-typescript
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      # - name: Extract SpacetimeDB branch name from file
+      #   id: extract-branch
+      #   run: |
+      #     # Define the path to the branch file
+      #     BRANCH_FILE=".github/spacetimedb-branch.txt"
+
+      #     # Default to master if file doesn't exist
+      #     if [ ! -f "$BRANCH_FILE" ]; then
+      #       echo "::notice::No SpacetimeDB branch file found, using 'master'"
+      #       echo "branch=master" >> $GITHUB_OUTPUT
+      #       exit 0
+      #     fi
+
+      #     # Read and trim whitespace from the file
+      #     branch=$(cat "$BRANCH_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+
+      #     # Fallback to master if empty
+      #     if [ -z "$branch" ]; then
+      #       echo "::warning::SpacetimeDB branch file is empty, using 'master'"
+      #       branch="master"
+      #     fi
+
+      #     echo "branch=$branch" >> $GITHUB_OUTPUT
+      #     echo "Using SpacetimeDB branch from file: $branch"
+
+      - name: Install Rust toolchain
+        uses: dsherret/rust-toolchain-file@v1
+      - name: Set default rust toolchain
+        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
+
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the main CI job save the cache since it builds the most things
+          save-if: false
+          prefix-key: v1
+
+      # # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
+      # # ChatGPT suspects that this could be due to different build invocations using the same target dir,
+      # # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
+      # # `cargo build --manifest-path` (which apparently build different dependency trees).
+      # # However, we've been unable to fix it so... /shrug
+      # - name: Check v8 outputs
+      #   run: |
+      #     find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
+      #     if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
+      #       echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
+      #       cargo clean -p v8 || true
+      #       cargo build -p v8
+      #     fi
+      #     if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then
+      #       echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
+      #       cargo clean --release -p v8 || true
+      #       cargo build --release -p v8
+      #     fi
+
+      # - name: Install SpacetimeDB CLI from the local checkout
+      #   run: |
+      #     export CARGO_HOME="$HOME/.cargo"
+      #     echo "$CARGO_HOME/bin" >> "$GITHUB_PATH"
+      #     cargo install --force --path crates/cli --locked --message-format=short
+      #     cargo install --force --path crates/standalone --locked --message-format=short
+      #     # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
+      #     ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
+      #     # Clear any existing information
+      #     spacetime server clear -y
+
+      - name: Run TypeScript tests
+        run: cargo ci typescript-test
+
+      # - name: Run quickstart-chat tests
+      #   working-directory: examples/quickstart-chat
+      #   run: pnpm test
+      #
+      # # Run this step always, even if the previous steps fail
+      # - name: Print rows in the user table
+      #   if: always()
+      #   run: spacetime sql quickstart-chat "SELECT * FROM user"

.github/workflows/discord-posts.yml

@@ -20,6 +20,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y gh
 
+      # TODO: Perhaps we should merge this into the public-pr-merge.yml workflow, now that that exists.
       - name: Send Discord notification
         env:
           DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
@@ -69,3 +70,25 @@ jobs:
           # Use `jq` to construct the json data blob in the format required by the webhook.
           data="$(jq --null-input --arg msg "$message" '.content=$msg')"
           curl -X POST -H 'Content-Type: application/json' -d "$data" "${DISCORD_WEBHOOK_URL}"
+
+  invokePrivate:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true &&
+      github.event.pull_request.base.ref == 'master'
+    permissions:
+      contents: read
+    steps:
+      - name: Dispatch private merge workflow
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: 'clockworklabs',
+              repo: 'SpacetimeDBPrivate',
+              workflow_id: 'public-pr-merge.yml',
+              ref: 'master',
+              inputs: {
+                public_pr_number: String(context.payload.pull_request.number),
+              }
+            });