Merge branch 'master' into tyler/ts-sdk-improvements

Author: cloutiertyler

.github/workflows/ci.yml

@@ -22,11 +22,19 @@ jobs:
     name: Smoketests
     strategy:
       matrix:
+        runner: [spacetimedb-new-runner, windows-latest]
         include:
-          - { runner: spacetimedb-runner, smoketest_args: --docker }
-          - { runner: windows-latest, smoketest_args: --no-build-cli }
-        runner: [ spacetimedb-runner, windows-latest ]
+          - runner: spacetimedb-new-runner
+            smoketest_args: --docker
+            container:
+              image: localhost:5000/spacetimedb-ci:latest
+              options: --privileged
+          - runner: windows-latest
+            smoketest_args: --no-build-cli
+            container: null
     runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}
+    
     steps:
       - name: Find Git ref
         env:
@@ -63,6 +71,10 @@ jobs:
         if: runner.os == 'Windows'
         run: choco install psql -y --no-progress
         shell: powershell
+      - name: Start Docker daemon
+        if: runner.os == 'Linux'
+        run: /usr/local/bin/start-docker.sh
+
       - name: Build and start database (Linux)
         if: runner.os == 'Linux'
         run: docker compose up -d
@@ -93,7 +105,11 @@ jobs:
 
   test:
     name: Test Suite
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - name: Find Git ref
         env:
@@ -140,8 +156,9 @@ jobs:
         #Note: Unreal tests will be run separately
         run: cargo test --all -- --skip unreal
 
+        # The fallocate tests have been flakely when running in parallel
       - name: Run fallocate tests
-        run: cargo test -p spacetimedb-durability --features fallocate
+        run: cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1
 
       - name: Check that the test outputs are up-to-date
         run: bash tools/check-diff.sh
@@ -157,7 +174,11 @@ jobs:
 
   lints:
     name: Lints
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -196,7 +217,11 @@ jobs:
 
   wasm_bindings:
     name: Build and test wasm bindings
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - uses: actions/checkout@v3
 
@@ -221,7 +246,11 @@ jobs:
 
   publish_checks:
     name: Check that packages are publishable
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     permissions: read-all
     steps:
       - uses: actions/checkout@v3
@@ -289,7 +318,7 @@ jobs:
     name: Unreal Engine Tests
     # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
     # a custom runner.
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
     # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
     if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
     container:
@@ -368,13 +397,17 @@ jobs:
 
               cd "$GITHUB_WORKSPACE/sdks/unreal"
               cargo --version
-              cargo test
+              cargo test -- --test-threads=1
             '
 
   cli_docs:
     name: Check CLI docs
     permissions: read-all
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - name: Find Git ref
         env:

.github/workflows/csharp-test.yml

@@ -12,7 +12,12 @@ concurrency:
 
 jobs:
   unity-testsuite:
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
+        --cgroupns=host
     # Cancel any previous testsuites running on the same PR and/or ref.
     concurrency:
       group: unity-test-${{ github.event.pull_request.number || github.ref }}
@@ -81,7 +86,7 @@ jobs:
           cargo install --force --path crates/cli --locked --message-format=short
           cargo install --force --path crates/standalone --locked --message-format=short
           # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
-          ln -sf $HOME/.cargo/bin/spacetimedb-cli $HOME/.cargo/bin/spacetime
+          ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
         env:
           # Share the target directory with our local project to avoid rebuilding same SpacetimeDB crates twice.
           CARGO_TARGET_DIR: demo/Blackholio/server-rust/target
@@ -157,6 +162,9 @@ jobs:
           key: Unity-${{ github.head_ref }}
           restore-keys: Unity-
 
+        # We need this to support "Docker in Docker"
+      - name: Start Docker daemon
+        run: /usr/local/bin/start-docker.sh
       - name: Run Unity tests
         uses: game-ci/unity-test-runner@v4
         with:

.github/workflows/discord-posts.yml

@@ -27,6 +27,7 @@ jobs:
           PR_NUMBER: ${{ github.event.pull_request.number }}
           PR_URL: ${{ github.event.pull_request.html_url }}
           MENTION_ON_FAILURE: ${{ secrets.DEV_OPS_ROLE_ID }}
+          DISCORD_USER_MAP: ${{ secrets.DISCORD_USER_MAP }}
         run: |
           message="PR merged: [(#${PR_NUMBER}) ${PR_TITLE}](<${PR_URL}>)"
           # Note that anything besides success is treated as a failure (e.g. if the check did not run at all, or if it is still pending).
@@ -39,6 +40,18 @@ jobs:
               ' |
               jq -r '"\(.workflow) / \(.name): \(.state)"'
           )"
+
+          # Lookup PR author's Discord ID from the provided JSON map (if any)
+          author_discord_id="$(
+            jq -r \
+              --arg u "${{ github.event.pull_request.user.login }}" \
+              'try .[$u] catch empty' \
+              <<<"${DISCORD_USER_MAP}"
+          )"
+          if [ -z "${author_discord_id}" ]; then
+            echo "Warning: PR author not found not found in USER_LOOKUP_JSON"
+          fi
+
           message+=$'\n'
           if [[ -z "${FAILED_CHECKS}" ]]; then
             message+='All checks passed.'
@@ -47,7 +60,11 @@ jobs:
             message+=$'\n'
             # This uses special Discord syntax for pinging a particular role.
             # Note the '&' - this is the difference between pinging a *role* and pinging a *person*.
-            message+=" (cc <@&${MENTION_ON_FAILURE}>)"
+            message+=" (cc <@&${MENTION_ON_FAILURE}>"
+            if [[ -n "${author_discord_id}" ]]; then
+              message+=" <@${author_discord_id}>"
+            fi
+            message+=")"
           fi
           # Use `jq` to construct the json data blob in the format required by the webhook.
           data="$(jq --null-input --arg msg "$message" '.content=$msg')"

.github/workflows/internal-tests.yml

@@ -18,16 +18,20 @@ jobs:
     # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
     if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
     runs-on: ubuntu-latest
+    env:
+      TARGET_OWNER: clockworklabs
+      TARGET_REPO: SpacetimeDBPrivate
     steps:
-      - name: Trigger and wait for Internal Tests
+      - id: dispatch
+        name: Trigger tests
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
           script: |
-            const targetOwner = 'clockworklabs';
-            const targetRepo = 'SpacetimeDBPrivate';
             const workflowId = 'ci.yml';
             const targetRef = 'master';
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
             // Use the ref for pull requests because the head sha is brittle (github does some extra dance where it merges in master).
             const publicRef = (context.eventName === 'pull_request') ? context.payload.pull_request.head.ref : context.sha;
             const preDispatch = new Date().toISOString();
@@ -48,12 +52,10 @@ jobs:
 
             const sleep = (ms) => new Promise(r => setTimeout(r, ms));
 
-            // Wait for workflow to be kicked off
-            await sleep(30000);
-
             // Find the dispatched run by name
             let runId = null;
             for (let attempt = 0; attempt < 20 && !runId; attempt++) { // up to ~10 minutes to locate the run
+              await sleep(5000);
               const runsResp = await github.rest.actions.listWorkflowRuns({
                 owner: targetOwner,
                 repo: targetRepo,
@@ -72,7 +74,6 @@ jobs:
                 runId = candidates[0].id;
                 break;
               }
-              await sleep(30000);
             }
 
             if (!runId) {
@@ -83,9 +84,23 @@ jobs:
             // Provide direct link and context prior to waiting
             const runUrl = `https://github.com/${targetOwner}/${targetRepo}/actions/runs/${runId}`;
             core.info(`View run: ${runUrl}`);
-            core.info('Waiting for workflow result...');
 
-            // Wait for completion
+            core.setOutput('run_id', String(runId));
+            core.setOutput('run_url', runUrl);
+
+      - name: Wait for Internal Tests to complete
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
+            const runUrl = `${{ steps.dispatch.outputs.run_url }}`;
+            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+            core.info(`Waiting for workflow result... ${runUrl}`);
+
             let conclusion = null;
             for (let attempt = 0; attempt < 240; attempt++) { // up to ~2 hours
               const runResp = await github.rest.actions.getWorkflowRun({
@@ -110,3 +125,24 @@ jobs:
             if (conclusion !== 'success') {
               core.setFailed(`Private workflow failed with conclusion: ${conclusion}`);
             }
+
+      - name: Cancel invoked run if workflow cancelled
+        if: ${{ cancelled() }}
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
+            if (!runId) {
+              core.warning('No run_id available to cancel.');
+              return;
+            }
+            core.info(`Cancelling private workflow run ${runId} in ${targetOwner}/${targetRepo}...`);
+            await github.rest.actions.cancelWorkflowRun({
+              owner: targetOwner,
+              repo: targetRepo,
+              run_id: runId,
+            });
+            core.info('Cancellation requested.');

Cargo.lock

@@ -178,6 +178,7 @@ enum-map = "2.6.3"
 env_logger = "0.10"
 ethnum = { version = "1.5.0", features = ["serde"] }
 flate2 = "1.0.24"
+flume = { version = "0.11", default-features = false, features = ["async"] }
 foldhash = "0.1.4"
 fs-err = "2.9.0"
 fs_extra = "1.3.0"

Cargo.toml

@@ -1,8 +1,9 @@
 mod module_bindings;
 use module_bindings::*;
 use std::env;
+use std::io::{self, Read};
 
-use spacetimedb_sdk::{DbConnection, Table};
+use spacetimedb_sdk::{DbContext, Table};
 
 fn main() {
     // The URI of the SpacetimeDB instance hosting our chat module.
@@ -14,28 +15,31 @@ fn main() {
     // Connect to the database
     let conn = DbConnection::builder()
         .with_module_name(db_name)
-        .with_host(host)
+        .with_uri(host)
         .on_connect(|_, _, _| {
             println!("Connected to SpacetimeDB");
         })
-        .on_connect_error(|e| {
+        .on_connect_error(|_ctx, e| {
             eprintln!("Connection error: {:?}", e);
             std::process::exit(1);
         })
         .build()
         .expect("Failed to connect");
 
+    conn.run_threaded();
+
     // Subscribe to the person table
-    conn.subscribe(&[
-        "SELECT * FROM person"
-    ]);
+    conn.subscription_builder()
+        .on_applied(|_ctx| println!("Subscripted to the person table"))
+        .on_error(|_ctx, e| eprintln!("There was an error when subscring to the person table: {e}"))
+        .subscribe(["SELECT * FROM person"]);
 
     // Register a callback for when rows are inserted into the person table
-    Person::on_insert(|_ctx, person| {
+    conn.db().person().on_insert(|_ctx, person| {
         println!("New person: {}", person.name);
     });
 
-    // Run the connection on the current thread
-    // This will block and handle all database events
-    conn.run();
+    println!("Press any key to exit...");
+
+    let _ = io::stdin().read(&mut [0u8]).unwrap();
 }

crates/cli/templates/basic-rust/client/src/main.rs

@@ -1,6 +1,6 @@
 use spacetimedb::{ReducerContext, Table};
 
-#[spacetimedb::table(name = person)]
+#[spacetimedb::table(name = person, public)]
 pub struct Person {
     name: String,
 }