Merge branch 'master' into jdetter/move-docs-to-new-runner

Author: jdetter

.github/Dockerfile

@@ -0,0 +1,6 @@
+# Minimal Dockerfile that just wraps pre-built binaries, so we can test the server inside docker
+FROM rust:1.90.0
+RUN mkdir -p /stdb/data
+COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli /usr/local/bin/
+COPY ./crates/standalone/config.toml /stdb/data/config.toml
+RUN ln -s /usr/local/bin/spacetimedb-cli /usr/local/bin/spacetime

.github/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  node:
+    labels:
+      app: spacetimedb
+    build:
+      context: ../
+      dockerfile: .github/Dockerfile
+    ports:
+      - "3000:3000"
+      # Postgres
+      - "5432:5432"
+    entrypoint: spacetime start --pg-port 5432
+    privileged: true
+    environment:
+      RUST_BACKTRACE: 1

.github/workflows/ci.yml

@@ -34,7 +34,8 @@ jobs:
             container: null
     runs-on: ${{ matrix.runner }}
     container: ${{ matrix.container }}
-    
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -53,6 +54,15 @@ jobs:
         with:
           ref: ${{ env.GIT_REF }}
       - uses: dsherret/rust-toolchain-file@v1
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          cache-on-failure: true
+          cache-all-crates: true
+          cache-workspace-crates: true
+
       - uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
@@ -71,21 +81,25 @@ jobs:
         if: runner.os == 'Windows'
         run: choco install psql -y --no-progress
         shell: powershell
+      - name: Build crates
+        run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
       - name: Start Docker daemon
         if: runner.os == 'Linux'
         run: /usr/local/bin/start-docker.sh
 
       - name: Build and start database (Linux)
         if: runner.os == 'Linux'
-        run: docker compose up -d
+        run: |
+          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
+          rm .dockerignore
+          docker compose -f .github/docker-compose.yml up -d
       - name: Build and start database (Windows)
         if: runner.os == 'Windows'
         run: |
           # Fail properly if any individual command fails
           $ErrorActionPreference = 'Stop'
           $PSNativeCommandUseErrorActionPreference = $true
 
-          cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
           Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
           cd modules
           # the sdk-manifests on windows-latest are messed up, so we need to update them
@@ -95,13 +109,13 @@ jobs:
         with: { python-version: '3.12' }
         if: runner.os == 'Windows'
       - name: Install python deps
-        run: python -m pip install psycopg2-binary xmltodict
+        run: python -m pip install -r smoketests/requirements.txt
       - name: Run smoketests
         # Note: clear_database and replication only work in private
         run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
-        run: docker compose down
+        run: docker compose -f .github/docker-compose.yml down
 
   test:
     name: Test Suite
@@ -110,6 +124,8 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -129,6 +145,13 @@ jobs:
           ref: ${{ env.GIT_REF }}
 
       - uses: dsherret/rust-toolchain-file@v1
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
 
       - uses: actions/setup-dotnet@v3
         with:
@@ -179,13 +202,23 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
       - uses: dsherret/rust-toolchain-file@v1
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - uses: actions/setup-dotnet@v3
         with:
           global-json-file: global.json
@@ -222,12 +255,22 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - uses: actions/checkout@v3
 
       - uses: dsherret/rust-toolchain-file@v1
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - name: Run bindgen tests
         run: cargo test -p spacetimedb-codegen
 
@@ -408,6 +451,8 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -451,6 +496,14 @@ jobs:
 
       - uses: dsherret/rust-toolchain-file@v1
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - name: Check for docs change
         run: |
           cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md

.github/workflows/csharp-test.yml

@@ -23,6 +23,8 @@ jobs:
       group: unity-test-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     timeout-minutes: 30
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Checkout repository
         id: checkout-stdb
@@ -70,26 +72,18 @@ jobs:
 
       - name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
-        id: cache-rust-deps
         with:
-          workspaces: demo/Blackholio/server-rust
-          key: ${{ steps.checkout-stdb.outputs.commit }}
-          # Cache Rust deps even if unit tests have failed.
-          cache-on-failure: true
-          # Cache the CLI as well.
-          cache-all-crates: true
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the main CI job save the cache since it builds the most things
+          save-if: false
 
       - name: Install SpacetimeDB CLI from the local checkout
-        # Rebuild only if we didn't get a precise cache hit.
-        if: steps.cache-rust-deps.outputs.cache-hit == 'false'
         run: |
           cargo install --force --path crates/cli --locked --message-format=short
           cargo install --force --path crates/standalone --locked --message-format=short
           # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
           ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
-        env:
-          # Share the target directory with our local project to avoid rebuilding same SpacetimeDB crates twice.
-          CARGO_TARGET_DIR: demo/Blackholio/server-rust/target
 
       - name: Check quickstart-chat bindings are up to date
         working-directory: sdks/csharp

.github/workflows/typescript-lint.yml

@@ -9,7 +9,11 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: --privileged
+
 
     steps:
       - name: Checkout repository

.github/workflows/typescript-test.yml

@@ -13,7 +13,12 @@ concurrency:
 
 jobs:
   build-and-test:
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
 
     steps:
       - name: Checkout repository
@@ -81,18 +86,18 @@ jobs:
       - name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
         with:
-          workspaces: |
-            .
-            modules/quickstart-chat
-          shared-key: quickstart-chat-test
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the main CI job save the cache since it builds the most things
+          save-if: false
 
       - name: Install SpacetimeDB CLI from the local checkout
         run: |
           cargo install --force --path crates/cli --locked --message-format=short
           cargo install --force --path crates/standalone --locked --message-format=short
           # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
-          rm -f $HOME/.cargo/bin/spacetime
-          ln -s $HOME/.cargo/bin/spacetimedb-cli $HOME/.cargo/bin/spacetime
+          rm -f $CARGO_HOME/bin/spacetime
+          ln -s $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
           # Clear any existing information
           spacetime server clear -y
 

.gitignore

@@ -226,3 +226,6 @@ new.json
 
 # Symlinked output from `nix build`
 result
+
+# Python venv directories
+venv/

crates/auth/src/identity.rs

@@ -4,6 +4,7 @@ pub use jsonwebtoken::{DecodingKey, EncodingKey};
 use serde::Deserializer;
 use serde::{Deserialize, Serialize};
 use spacetimedb_lib::Identity;
+use std::collections::HashMap;
 use std::time::SystemTime;
 
 #[derive(Debug, Clone)]
@@ -41,6 +42,9 @@ pub struct SpacetimeIdentityClaims {
     pub iat: SystemTime,
     #[serde_as(as = "Option<serde_with::TimestampSeconds>")]
     pub exp: Option<SystemTime>,
+
+    #[serde(flatten)]
+    pub extra: Option<HashMap<String, serde_json::Value>>,
 }
 
 fn deserialize_audience<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
@@ -84,6 +88,10 @@ pub struct IncomingClaims {
     pub iat: SystemTime,
     #[serde_as(as = "Option<serde_with::TimestampSeconds>")]
     pub exp: Option<SystemTime>,
+
+    /// All remaining claims from the JWT payload
+    #[serde(flatten)]
+    pub extra: Option<HashMap<String, serde_json::Value>>,
 }
 
 impl TryInto<SpacetimeIdentityClaims> for IncomingClaims {
@@ -122,6 +130,7 @@ impl TryInto<SpacetimeIdentityClaims> for IncomingClaims {
             audience: self.audience,
             iat: self.iat,
             exp: self.exp,
+            extra: self.extra,
         })
     }
 }

crates/bindings-csharp/BSATN.Runtime/BSATN/Runtime.cs

@@ -228,6 +228,12 @@ public void Write(BinaryWriter writer, Inner? value)
 
     public AlgebraicType GetAlgebraicType(ITypeRegistrar registrar) =>
         AlgebraicType.MakeOption(innerRW.GetAlgebraicType(registrar));
+
+    // Return a List BSATN serializer that can serialize this option as an array
+    public static List<Inner, InnerRW> GetListSerializer()
+    {
+        return new List<Inner, InnerRW>();
+    }
 }
 
 public readonly struct Bool : IReadWrite<bool>

crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/ExtraCompilationErrors.verified.txt

@@ -233,7 +233,7 @@ public partial struct TestDefaultFieldValues
 
             var returnValue = Module.ViewDefWrongContext((SpacetimeDB.ViewContext)ctx);
                                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-            using var output = new System.IO.MemoryStream();
+                SpacetimeDB.BSATN.List<Player, Player.BSATN> returnRW = new();
 */
     Message: Argument 1: cannot convert from 'SpacetimeDB.ViewContext' to 'SpacetimeDB.ReducerContext',
     Severity: Error,
@@ -279,7 +279,7 @@ SpacetimeDB.Internal.Module.RegisterClientVisibilityFilter(global::Module.MY_THI
 
             var returnValue = Module.ViewDefNoContext((SpacetimeDB.ViewContext)ctx);
                                      ^^^^^^^^^^^^^^^^
-            using var output = new System.IO.MemoryStream();
+                SpacetimeDB.BSATN.List<Player, Player.BSATN> returnRW = new();
 */
     Message: No overload for method 'ViewDefNoContext' takes 1 arguments,
     Severity: Error,
@@ -299,10 +299,10 @@ SpacetimeDB.Internal.Module.RegisterClientVisibilityFilter(global::Module.MY_THI
     }
   },
   {/*
-    
-    private static readonly SpacetimeDB.BSATN.ValueOption<NotSpacetimeType, NotSpacetimeType.BSATN> returnRW = new();
-                                                                                             ^^^^^
-    
+    Params: [],
+    ReturnType: new SpacetimeDB.BSATN.ValueOption<NotSpacetimeType, NotSpacetimeType.BSATN>().GetAlgebraicType(registrar)
+                                                                                     ^^^^^
+);
 */
     Message: The type name 'BSATN' does not exist in the type 'NotSpacetimeType',
     Severity: Error,
@@ -322,10 +322,10 @@ SpacetimeDB.Internal.Module.RegisterClientVisibilityFilter(global::Module.MY_THI
     }
   },
   {/*
-    Params: [],
-    ReturnType: new SpacetimeDB.BSATN.ValueOption<NotSpacetimeType, NotSpacetimeType.BSATN>().GetAlgebraicType(registrar)
-                                                                                     ^^^^^
-);
+            var returnValue = Module.ViewDefReturnsNotASpacetimeType((SpacetimeDB.AnonymousViewContext)ctx);
+                var listSerializer = SpacetimeDB.BSATN.ValueOption<NotSpacetimeType, NotSpacetimeType.BSATN>.GetListSerializer();
+                                                                                                      ^^^^^
+    var listValue = ModuleRegistration.ToListOrEmpty(returnValue);
 */
     Message: The type name 'BSATN' does not exist in the type 'NotSpacetimeType',
     Severity: Error,