Merge branch 'master' into noa/unconflate-budget-and-energy

Author: coolreader18

.github/docker-compose.yml

@@ -760,6 +760,131 @@ jobs:
           UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
           UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
 
+  godot-testsuite:
+    needs: [lints]
+    permissions:
+      contents: read
+    runs-on: spacetimedb-new-runner-2
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      UseLocalBsatnRuntime: true
+    steps:
+      - name: Checkout repository
+        id: checkout-stdb
+        uses: actions/checkout@v4
+
+      - name: Setup dotnet
+        uses: actions/setup-dotnet@v3
+        with:
+          global-json-file: global.json
+
+      - name: Override NuGet packages
+        run: |
+          dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime
+          dotnet pack -c Release crates/bindings-csharp/Runtime
+
+          # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository
+          # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if
+          # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages.
+          # This means that (if version numbers match) we will test the local versions of the C# packages, even
+          # if they're not pushed to NuGet.
+          # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file.
+          cd sdks/csharp
+          ./tools~/write-nuget-config.sh ../..
+
+      - name: Restore .NET solution
+        working-directory: sdks/csharp
+        run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln
+
+      # Now, setup the Godot tests.
+      - name: Patch spacetimedb dependency in Cargo.toml
+        working-directory: demo/Blackholio/server-rust
+        run: |
+          sed -i "s|spacetimedb *=.*|spacetimedb = \{ path = \"../../../crates/bindings\" \}|" Cargo.toml
+          cat Cargo.toml
+
+      - name: Install Rust toolchain
+        uses: dsherret/rust-toolchain-file@v1
+      - name: Set default rust toolchain
+        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
+
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the main CI job save the cache since it builds the most things
+          save-if: false
+          prefix-key: v1
+
+      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
+      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
+      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
+      # `cargo build --manifest-path` (which apparently build different dependency trees).
+      # However, we've been unable to fix it so... /shrug
+      - name: Check v8 outputs
+        run: |
+          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
+          if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then
+            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
+            cargo clean --release -p v8 || true
+            cargo build --release -p v8
+          fi
+
+      - name: Install SpacetimeDB CLI from the local checkout
+        run: |
+          export CARGO_HOME="$HOME/.cargo"
+          echo "$CARGO_HOME/bin" >> "$GITHUB_PATH"
+          cargo install --force --path crates/cli --locked --message-format=short
+          cargo install --force --path crates/standalone --locked --message-format=short
+          # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
+          ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
+
+      - name: Generate client bindings
+        working-directory: demo/Blackholio/server-rust
+        run: bash ./generate.sh -y
+
+      - name: Check for changes
+        run: |
+          tools/check-diff.sh demo/Blackholio/client-godot/module_bindings || {
+              echo 'Error: Godot bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`.'
+              exit 1
+          }
+
+      - name: Patch SpacetimeDB Godot SDK dependency
+        working-directory: demo/Blackholio/client-godot
+        run: |
+          dotnet remove package SpacetimeDB.ClientSDK.Godot
+          dotnet add reference ../../../sdks/csharp/SpacetimeDB.ClientSDK.Godot.csproj
+          cat blackholio.csproj
+
+      - name: Setup Godot
+        uses: chickensoft-games/setup-godot@v2
+        with:
+          version: 4.6.2
+          use-dotnet: true
+
+      - name: Restore Godot project
+        working-directory: demo/Blackholio/client-godot
+        run: dotnet restore --configfile ../../../NuGet.Config blackholio.csproj
+
+      - name: Build Godot project
+        run: godot --headless --verbose --path demo/Blackholio/client-godot --build-solutions --quit
+
+      - name: Start SpacetimeDB
+        run: |
+          spacetime start &
+          disown
+
+      - name: Publish godot-tests module to SpacetimeDB
+        working-directory: demo/Blackholio/server-rust
+        run: |
+          spacetime login --server-issued-login local
+          bash ./publish.sh
+
+      - name: Run Godot tests
+        run: godot --headless --path demo/Blackholio/client-godot --scene res://tests/GodotPlayModeTests.tscn
+
   csharp-testsuite:
     needs: [lints]
     runs-on: spacetimedb-new-runner-2
@@ -771,8 +896,6 @@ jobs:
         id: checkout-stdb
         uses: actions/checkout@v4
 
-      # Run cheap .NET tests first. If those fail, no need to run expensive Unity tests.
-
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:

.github/workflows/ci.yml

@@ -1 +1 @@
-../../licenses/BSL.txt
+../../licenses/apache2.txt

crates/bindings-macro/LICENSE

@@ -1 +1 @@
-../../licenses/BSL.txt
+../../licenses/apache2.txt

crates/bindings-sys/LICENSE

@@ -28,9 +28,24 @@ export async function decompress(
   const decompressedStream = readableStream.pipeThrough(decompressionStream);
 
   // Collect the decompressed chunks efficiently
-  const chunks = [];
-  for await (const chunk of decompressedStream) {
-    chunks.push(chunk);
+  const reader = decompressedStream.getReader();
+  const chunks: Uint8Array[] = [];
+  let totalLength = 0;
+  let result: any;
+
+  while (!(result = await reader.read()).done) {
+    chunks.push(result.value);
+    totalLength += result.value.length;
   }
-  return new Blob(chunks).bytes();
+
+  // Allocate a single Uint8Array for the decompressed data
+  const decompressedArray = new Uint8Array(totalLength);
+  let chunkOffset = 0;
+
+  for (const chunk of chunks) {
+    decompressedArray.set(chunk, chunkOffset);
+    chunkOffset += chunk.length;
+  }
+
+  return decompressedArray;
 }

crates/bindings-typescript/src/sdk/decompress.ts

@@ -1 +1 @@
-../../licenses/BSL.txt
+../../licenses/apache2.txt

crates/bindings/LICENSE

@@ -16,7 +16,9 @@ use spacetimedb_datastore::locking_tx_datastore::datastore::TxMetrics;
 use spacetimedb_datastore::locking_tx_datastore::state_view::{
     IterByColEqMutTx, IterByColRangeMutTx, IterMutTx, StateView,
 };
-use spacetimedb_datastore::locking_tx_datastore::{ApplyHistoryCounters, IndexScanPointOrRange, MutTxId, TxId};
+use spacetimedb_datastore::locking_tx_datastore::{
+    ApplyHistoryCounters, IndexScanPointOrRange, MutTxId, TxId, ViewCallInfo,
+};
 use spacetimedb_datastore::system_tables::{
     system_tables, StModuleRow, ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_SUB_ID,
 };
@@ -1601,6 +1603,26 @@ impl RelationalDB {
         Ok(())
     }
 
+    /// Materialize a view call and replace its committed read set on transaction commit.
+    ///
+    /// The read set is only marked for replacement after materialization succeeds. If the
+    /// transaction rolls back, the replacement marker rolls back with it.
+    pub fn materialize_view_call(
+        &self,
+        tx: &mut MutTxId,
+        table_id: TableId,
+        view_call: ViewCallInfo,
+        rows: Vec<ProductValue>,
+    ) -> Result<(), DBError> {
+        match view_call.sender {
+            Some(sender) => self.materialize_view(tx, table_id, sender, rows)?,
+            None => self.materialize_anonymous_view(tx, table_id, rows)?,
+        }
+        tx.replace_view_read_set(view_call);
+
+        Ok(())
+    }
+
     fn write_view_rows(
         &self,
         tx: &mut MutTxId,

crates/core/src/db/relational_db.rs

@@ -1088,6 +1088,66 @@ pub struct CallViewParams {
     pub timestamp: Timestamp,
 }
 
+pub(crate) struct ResolvedViewForRefresh<'a> {
+    pub view_id: ViewId,
+    pub table_id: TableId,
+    pub view_def: &'a ViewDef,
+}
+
+/// Lookup a module's [`ViewDef`] and check for consistency among
+/// its readset, `st_view`, and the [`ModuleDef`].
+pub(crate) fn resolve_view_for_refresh<'a>(
+    tx: &MutTxId,
+    module_def: &'a ModuleDef,
+    view_call: &ViewCallInfo,
+) -> anyhow::Result<ResolvedViewForRefresh<'a>> {
+    let st_view = tx
+        .lookup_st_view(view_call.view_id)
+        .with_context(|| format!("failed to look up view {}", view_call.view_id))?;
+
+    let view_id = st_view.view_id;
+    let table_id = st_view
+        .table_id
+        .ok_or_else(|| anyhow::anyhow!("view {:?} does not have a backing table", view_id))?;
+
+    let view_name: Identifier = st_view.view_name.into();
+    let view_def = module_def.view(&view_name).ok_or_else(|| {
+        anyhow::anyhow!(
+            "view `{}` for view id `{}` not found in current module",
+            view_name,
+            view_id
+        )
+    })?;
+
+    let is_anonymous = view_call.sender.is_none();
+
+    if st_view.is_anonymous != is_anonymous {
+        return Err(anyhow::anyhow!(
+            "found is_anonymous={} in st_view, but {} in readset when updating view `{}`",
+            st_view.is_anonymous,
+            is_anonymous,
+            view_name,
+        ));
+    }
+
+    let is_anonymous = view_def.is_anonymous;
+
+    if st_view.is_anonymous != is_anonymous {
+        return Err(anyhow::anyhow!(
+            "found is_anonymous={} in st_view, but {} in module when updating view `{}`",
+            st_view.is_anonymous,
+            is_anonymous,
+            view_name,
+        ));
+    }
+
+    Ok(ResolvedViewForRefresh {
+        view_id,
+        table_id,
+        view_def,
+    })
+}
+
 pub struct CallProcedureParams {
     pub timestamp: Timestamp,
     pub caller_identity: Identity,
@@ -2896,17 +2956,21 @@ impl ModuleHost {
         let mut total_duration = Duration::ZERO;
         let mut abi_duration = Duration::ZERO;
         let mut trapped = false;
-        for ViewCallInfo {
-            view_id,
-            table_id,
-            fn_ptr,
-            sender,
-        } in tx.views_for_refresh().cloned().collect::<Vec<_>>()
-        {
-            let Some(view_def) = module_def.get_view_by_id(fn_ptr, sender.is_none()) else {
-                outcome = ViewOutcome::Failed(format!("view with fn_ptr `{fn_ptr}` not found"));
-                break;
+        for view_call in tx.views_for_refresh().cloned().collect::<Vec<_>>() {
+            let sender = view_call.sender;
+            let resolved = match resolve_view_for_refresh(&tx, module_def, &view_call) {
+                Ok(resolved) => resolved,
+                Err(err) => {
+                    outcome = ViewOutcome::Failed(format!("failed to resolve view: {err}"));
+                    break;
+                }
             };
+            let ResolvedViewForRefresh {
+                view_id,
+                table_id,
+                view_def,
+            } = resolved;
+            let view_name = &view_def.name;
             let args = match FunctionArgs::Nullary.into_tuple_for_def(module_def, view_def) {
                 Ok(args) => args,
                 Err(err) => {
@@ -2918,7 +2982,7 @@ impl ModuleHost {
             let (result, trap) = Self::call_view_inner(
                 instance,
                 tx,
-                &view_def.name,
+                view_name,
                 view_id,
                 table_id,
                 view_def.fn_ptr,