Merge remote-tracking branch 'origin/master' into kim/ts/confirmed-reads

Author: kim

.github/workflows/ci.yml

@@ -103,9 +103,10 @@ jobs:
         run: |
           sudo mkdir /stdb
           sudo chmod 777 /stdb
-
+      
       - name: Run cargo test
-        run: cargo test --all
+        #Note: Unreal tests will be run separately
+        run: cargo test --all -- --skip unreal
 
       - name: Check that the test outputs are up-to-date
         run: bash tools/check-diff.sh
@@ -243,6 +244,90 @@ jobs:
           cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
           "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help
 
+  unreal_engine_tests:
+    name: Unreal Engine Tests
+    # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
+    # a custom runner.
+    runs-on: spacetimedb-runner
+    container:
+      image: ghcr.io/epicgames/unreal-engine:dev-5.6
+      credentials:
+        # Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and
+        # it seems to be totally happy.
+        # However, the token needs to be for a user that has access to the EpicGames org (see
+        # https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6)
+        username: ${{ github.actor }}
+        password: ${{ secrets.GHCR_TOKEN }}
+      # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run
+      # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle.
+      options: --user 0:0
+    steps:
+# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back
+# commits though.
+#      - name: Find Git ref
+#        env:
+#          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#        shell: bash
+#        run: |
+#          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+#          if test -n "${PR_NUMBER}"; then
+#            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+#          else
+#            GIT_REF="${{ github.ref }}"
+#          fi
+#          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Run Unreal Engine tests
+        working-directory: sdks/unreal
+        env:
+          UE_ROOT_PATH: /home/ue4/UnrealEngine
+        run: |
+
+          apt-get update
+          apt-get install -y acl curl ca-certificates
+
+          REPO="$GITHUB_WORKSPACE"
+          # Let ue4 read/write the workspace & tool caches without changing ownership
+          for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do
+            [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true
+          done
+
+          # Rust tool caches live under the runner tool cache so they persist
+          export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo"
+          export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup"
+          mkdir -p "$CARGO_HOME" "$RUSTUP_HOME"
+          chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME"
+
+          # Make sure the UE build script is executable (and parents traversable)
+          UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}"
+          chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true
+          chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true
+
+          # Run the build & tests as ue4 (who owns the UE tree)
+          sudo -E -H -u ue4 env \
+            HOME=/home/ue4 \
+            XDG_CONFIG_HOME=/home/ue4/.config \
+            CARGO_HOME="$CARGO_HOME" \
+            RUSTUP_HOME="$RUSTUP_HOME" \
+            PATH="$CARGO_HOME/bin:$PATH" \
+            bash -lc '
+              set -euxo pipefail
+              # Install rustup for ue4 if needed (uses the shared caches)
+              if ! command -v cargo >/dev/null 2>&1; then
+                curl -sSf https://sh.rustup.rs | sh -s -- -y
+              fi
+              rustup show >/dev/null
+              git config --global --add safe.directory "$GITHUB_WORKSPACE" || true
+
+              cd "$GITHUB_WORKSPACE/sdks/unreal"
+              cargo --version
+              cargo test
+            '
+
   cli_docs:
     name: Check CLI docs
     permissions: read-all
@@ -277,7 +362,6 @@ jobs:
 
       - name: Get pnpm store directory
         shell: bash
-        working-directory: sdks/typescript
         run: |
           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
 

.github/workflows/typescript-lint.yml

@@ -25,7 +25,6 @@ jobs:
           run_install: true
 
       - name: Get pnpm store directory
-        working-directory: sdks/typescript
         shell: bash
         run: |
           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
@@ -39,5 +38,4 @@ jobs:
             ${{ runner.os }}-pnpm-store-
 
       - name: Lint
-        working-directory: sdks/typescript
         run: pnpm lint

.github/workflows/typescript-test.yml

@@ -26,7 +26,7 @@ jobs:
 
       - name: Get pnpm store directory
         shell: bash
-        working-directory: sdks/typescript
+        working-directory: crates/bindings-typescript
         run: |
           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
 
@@ -38,22 +38,14 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
       
-      - name: Build module library
+      - name: Build module library and SDK
         working-directory: crates/bindings-typescript
         run: pnpm build
 
-      - name: Run module library tests
+      - name: Run module library and SDK tests
         working-directory: crates/bindings-typescript 
         run: pnpm test
 
-      - name: Build SDK
-        working-directory: sdks/typescript
-        run: pnpm build
-
-      - name: Run SDK tests
-        working-directory: sdks/typescript/packages/sdk
-        run: pnpm test
-
       # - name: Extract SpacetimeDB branch name from file
       #   id: extract-branch
       #   run: |
@@ -104,12 +96,12 @@ jobs:
       - name: Generate client bindings
         working-directory: modules/quickstart-chat
         run: |
-          spacetime generate --lang typescript --out-dir ../../sdks/typescript/examples/quickstart-chat/src/module_bindings
-          cd ../../sdks/typescript
-          pnpm lint --write
+          spacetime generate --lang typescript --out-dir ../../crates/bindings-typescript/examples/quickstart-chat/src/module_bindings
+          cd ../../crates/bindings-typescript
+          pnpm format
 
       - name: Check for changes
-        working-directory: sdks/typescript
+        working-directory: crates/bindings-typescript
         run: |
           # This was copied from SpacetimeDB/tools/check-diff.sh.
           # It's required because `spacetime generate` creates lines with the SpacetimeDB commit
@@ -137,7 +129,7 @@ jobs:
       #     spacetime logs quickstart-chat
 
       - name: Check that quickstart-chat builds
-        working-directory: sdks/typescript/examples/quickstart-chat
+        working-directory: crates/bindings-typescript/examples/quickstart-chat
         run: pnpm build
 
       # - name: Run quickstart-chat tests

.gitignore

@@ -216,3 +216,6 @@ new.json
 # Keys
 *.pem
 .ok.sql
+
+# Test data
+!crates/core/testdata/

.prettierignore

@@ -3,3 +3,4 @@ pnpm-lock.yaml
 dist
 target
 .github
+coverage

Cargo.lock

@@ -26,6 +26,7 @@ members = [
   "crates/sats",
   "crates/schema",
   "sdks/rust",
+  "sdks/unreal",
   "crates/snapshot",
   "crates/sqltest",
   "crates/sql-parser",
@@ -47,7 +48,7 @@ members = [
   "sdks/rust/tests/connect_disconnect_client",
   "tools/upgrade-version",
   "tools/license-check",
-  "sdks/typescript/packages/test-app/server",
+  "crates/bindings-typescript/test-app/server",
 ]
 default-members = ["crates/cli", "crates/standalone", "crates/update"]
 # cargo feature graph resolver. v3 is default in edition2024 but workspace

Cargo.toml

@@ -21,7 +21,7 @@ Additional Use Grant: You may make use of the Licensed Work provided your
                       Licensed Work by creating tables whose schemas are
                       controlled by such third parties.
 
-Change Date:          2030-09-03
+Change Date:          2030-09-22
 
 Change License:       GNU Affero General Public License v3.0 with a linking
                       exception

LICENSE.txt

@@ -11,6 +11,7 @@ spacetimedb-lib = { workspace = true, features = ["serde"] }
 
 anyhow.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 serde_with.workspace = true
 jsonwebtoken.workspace = true
 

crates/auth/Cargo.toml

@@ -6,9 +6,27 @@ use serde::{Deserialize, Serialize};
 use spacetimedb_lib::Identity;
 use std::time::SystemTime;
 
+#[derive(Debug, Clone)]
+pub struct ConnectionAuthCtx {
+    pub claims: SpacetimeIdentityClaims,
+    pub jwt_payload: String,
+}
+
+impl TryFrom<SpacetimeIdentityClaims> for ConnectionAuthCtx {
+    type Error = anyhow::Error;
+    fn try_from(claims: SpacetimeIdentityClaims) -> Result<Self, Self::Error> {
+        let payload =
+            serde_json::to_string(&claims).map_err(|e| anyhow::anyhow!("Failed to serialize claims: {}", e))?;
+        Ok(ConnectionAuthCtx {
+            claims,
+            jwt_payload: payload,
+        })
+    }
+}
+
 // These are the claims that can be attached to a request/connection.
 #[serde_with::serde_as]
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct SpacetimeIdentityClaims {
     #[serde(rename = "hex_identity")]
     pub identity: Identity,