Merge branch 'master' into rekhoff/force-websocket-disconnect

Author: rekhoff

.cargo/config.toml

@@ -3,6 +3,7 @@ rustflags = ["--cfg", "tokio_unstable"]
 
 [alias]
 bump-versions = "run -p upgrade-version --"
+llm = "run --package xtask-llm-benchmark --bin llm_benchmark --"
 ci = "run -p ci --"
 
 [target.x86_64-pc-windows-msvc]

.gitattributes

@@ -1 +1,3 @@
 **/module_bindings/** linguist-generated=true eol=lf
+/docs/llms/** linguist-generated=true
+/docs/llms/*-details.json linguist-generated=false

.github/workflows/ci.yml

@@ -7,9 +7,9 @@ on:
   workflow_dispatch:
     inputs:
       pr_number:
-        description: 'Pull Request Number'
+        description: "Pull Request Number"
         required: false
-        default: ''
+        default: ""
 
 name: CI
 
@@ -19,6 +19,7 @@ concurrency:
 
 jobs:
   docker_smoketests:
+    needs: [lints]
     name: Smoketests
     strategy:
       matrix:
@@ -69,6 +70,14 @@ jobs:
         with:
           global-json-file: global.json
 
+      - name: Override NuGet packages
+        shell: bash
+        run: |
+          dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime
+          dotnet pack -c Release crates/bindings-csharp/Runtime
+          cd sdks/csharp
+          ./tools~/write-nuget-config.sh ../..
+
       # nodejs and pnpm are required for the typescript quickstart smoketest
       - name: Set up Node.js
         uses: actions/setup-node@v4
@@ -108,7 +117,7 @@ jobs:
           dotnet workload config --update-mode manifests
           dotnet workload update
       - uses: actions/setup-python@v5
-        with: { python-version: '3.12' }
+        with: { python-version: "3.12" }
         if: runner.os == 'Windows'
       - name: Install python deps
         run: python -m pip install -r smoketests/requirements.txt
@@ -120,6 +129,7 @@ jobs:
         run: docker compose -f .github/docker-compose.yml down
 
   test:
+    needs: [lints]
     name: Test Suite
     runs-on: spacetimedb-new-runner
     container:
@@ -500,19 +510,48 @@ jobs:
         run: |
           cargo ci cli-docs
 
+  llm_ci_check:
+    name: Verify LLM benchmark is up to date
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    # Disable the tests because they are causing us headaches with merge conflicts and re-runs etc.
+    if: false
+    steps:
+      # Build the tool from master to ensure consistent hash computation
+      # with the llm-benchmark-update workflow (which also uses master's tool).
+      - name: Checkout master (build tool from trusted code)
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          fetch-depth: 1
+
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Install llm-benchmark tool from master
+        run: |
+          cargo install --path tools/xtask-llm-benchmark --locked
+          command -v llm_benchmark
+
+      # Now checkout the PR branch to verify its benchmark files
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+        with:
+          clean: false
+
+      - name: Run hash check (both langs)
+        run: llm_benchmark ci-check
+
   unity-testsuite:
+    needs: [lints]
     # Skip if this is an external contribution.
     # The license secrets will be empty, so the step would fail anyway.
     if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
     permissions:
       contents: read
       checks: write
-    runs-on: spacetimedb-new-runner
-    container:
-      image: localhost:5000/spacetimedb-ci:latest
-      options: >-
-        --privileged
-        --cgroupns=host
+    runs-on: spacetimedb-unity-runner
     timeout-minutes: 30
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/target
@@ -542,6 +581,10 @@ jobs:
           cd sdks/csharp
           ./tools~/write-nuget-config.sh ../..
 
+      - name: Restore .NET solution
+        working-directory: sdks/csharp
+        run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln
+
       # Now, setup the Unity tests.
       - name: Patch spacetimedb dependency in Cargo.toml
         working-directory: demo/Blackholio/server-rust
@@ -579,13 +622,16 @@ jobs:
               exit 1
           }
 
+      - name: Hydrate Unity SDK DLLs
+        run: cargo ci dlls
+
       - name: Check Unity meta files
         uses: DeNA/unity-meta-check@v3
         with:
           enable_pr_comment: ${{ github.event_name == 'pull_request' }}
           target_path: sdks/csharp
         env:
-          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
       - name: Start SpacetimeDB
         run: |
@@ -601,10 +647,7 @@ jobs:
       - name: Patch com.clockworklabs.spacetimedbsdk dependency in manifest.json
         working-directory: demo/Blackholio/client-unity/Packages
         run: |
-          # Replace the com.clockworklabs.spacetimedbsdk dependency with the current branch.
-          # Note: Pointing to a local directory does not work, because our earlier steps nuke our meta files, which then causes Unity to not properly respect the DLLs (e.g.
-          # codegen does not work properly).
-          yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "https://github.com/clockworklabs/SpacetimeDB.git?path=sdks/csharp#${{ github.head_ref }}"' manifest.json
+          yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "file:../../../../sdks/csharp"' manifest.json
           cat manifest.json
 
       - uses: actions/cache@v3
@@ -613,9 +656,6 @@ jobs:
           key: Unity-${{ github.head_ref }}
           restore-keys: Unity-
 
-        # We need this to support "Docker in Docker"
-      - name: Start Docker daemon
-        run: /usr/local/bin/start-docker.sh
       - name: Run Unity tests
         uses: game-ci/unity-test-runner@v4
         with:
@@ -624,13 +664,14 @@ jobs:
           githubToken: ${{ secrets.GITHUB_TOKEN }}
           testMode: playmode
           useHostNetwork: true
-          artifactsPath: ''
+          artifactsPath: ""
         env:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
           UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
           UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
 
   csharp-testsuite:
+    needs: [lints]
     runs-on: spacetimedb-new-runner
     container:
       image: localhost:5000/spacetimedb-ci:latest
@@ -666,9 +707,13 @@ jobs:
           cd sdks/csharp
           ./tools~/write-nuget-config.sh ../..
 
+      - name: Restore .NET solution
+        working-directory: sdks/csharp
+        run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln
+
       - name: Run .NET tests
         working-directory: sdks/csharp
-        run: dotnet test -warnaserror
+        run: dotnet test -warnaserror --no-restore
 
       - name: Verify C# formatting
         working-directory: sdks/csharp
@@ -737,3 +782,129 @@ jobs:
               echo 'Error: Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`.'
               exit 1
           }
+
+  internal-tests:
+    name: Internal Tests
+    needs: [lints]
+    # Skip if not a PR or a push to master
+    # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
+    if: ${{ (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master'))
+      && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }}
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    env:
+      TARGET_OWNER: clockworklabs
+      TARGET_REPO: SpacetimeDBPrivate
+    steps:
+      - id: dispatch
+        name: Trigger tests
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const workflowId = 'ci.yml';
+            const targetRef = 'master';
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            // Use the ref for pull requests because the head sha is brittle (github does some extra dance where it merges in master).
+            const publicRef = (context.eventName === 'pull_request') ? context.payload.pull_request.head.ref : context.sha;
+            const preDispatch = new Date().toISOString();
+  
+            // Dispatch the workflow in the target repository
+            await github.rest.actions.createWorkflowDispatch({
+              owner: targetOwner,
+              repo: targetRepo,
+              workflow_id: workflowId,
+              ref: targetRef,
+              inputs: { public_ref: publicRef }
+            });
+  
+            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+  
+            // Find the dispatched run by name
+            let runId = null;
+            for (let attempt = 0; attempt < 20 && !runId; attempt++) { // up to ~10 minutes to locate the run
+              await sleep(5000);
+              const runsResp = await github.rest.actions.listWorkflowRuns({
+                owner: targetOwner,
+                repo: targetRepo,
+                workflow_id: workflowId,
+                event: 'workflow_dispatch',
+                branch: targetRef,
+                per_page: 50,
+              });
+  
+              const expectedName = `CI [public_ref=${publicRef}]`;
+              const candidates = runsResp.data.workflow_runs
+                .filter(r => r.name === expectedName && new Date(r.created_at) >= new Date(preDispatch))
+                .sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+  
+              if (candidates.length > 0) {
+                runId = candidates[0].id;
+                break;
+              }
+            }
+  
+            if (!runId) {
+              core.setFailed('Failed to locate dispatched run in the private repository.');
+              return;
+            }
+  
+            const runUrl = `https://github.com/${targetOwner}/${targetRepo}/actions/runs/${runId}`;
+            core.info(`View run: ${runUrl}`);
+            core.setOutput('run_id', String(runId));
+            core.setOutput('run_url', runUrl);
+  
+      - name: Wait for Internal Tests to complete
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
+            const runUrl = `${{ steps.dispatch.outputs.run_url }}`;
+            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+  
+            core.info(`Waiting for workflow result... ${runUrl}`);
+  
+            let conclusion = null;
+            for (let attempt = 0; attempt < 240; attempt++) { // up to ~2 hours
+              const runResp = await github.rest.actions.getWorkflowRun({
+                owner: targetOwner,
+                repo: targetRepo,
+                run_id: runId
+              });
+              const { status, conclusion: c } = runResp.data;
+              if (status === 'completed') {
+                conclusion = c || 'success';
+                break;
+              }
+              await sleep(30000);
+            }
+  
+            if (!conclusion) {
+              core.setFailed('Timed out waiting for private workflow to complete.');
+              return;
+            }
+  
+            if (conclusion !== 'success') {
+              core.setFailed(`Private workflow failed with conclusion: ${conclusion}`);
+            }
+  
+      - name: Cancel invoked run if workflow cancelled
+        if: ${{ cancelled() }}
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
+            if (!runId) return;
+            await github.rest.actions.cancelWorkflowRun({
+              owner: targetOwner,
+              repo: targetRepo,
+              run_id: runId,
+            });