[bfops/rust-smoketests-lib]: Merge remote-tracking branch 'origin/master' into bfops/rust-smoketests-lib

Author: bfops

.github/GREMLINS.md

@@ -0,0 +1,33 @@
+# GREMLINS.md — Who Lives in the Pipes
+
+This file documents the automated agents and bots that operate on this repository.
+
+## Clawd 🔧
+
+- **What:** Anti-entropy gremlin / CI watchdog
+- **GitHub account:** `clockwork-labs-bot`
+- **Discord channel:** #gremlins (CL - SpacetimeDB)
+- **Powered by:** [OpenClaw](https://github.com/openclaw/openclaw) + Claude
+
+### What Clawd does
+
+- **Monitors CI** — watches for failures, flaky tests, and regressions
+- **Reviews PRs** — comments on obvious bugs (never approves)
+- **Surfaces stale PRs** — finds approved PRs that just need a rebase
+- **Documents testing** — creates and maintains `DEVELOP.md` files explaining CI and test infrastructure
+- **Alerts the team** — posts findings in #gremlins, pings DevOps when something needs attention
+
+### What Clawd does NOT do
+
+- Approve or merge PRs
+- Take any destructive action (delete branches, close PRs, force push)
+- Modify production infrastructure
+
+### Contacting Clawd
+
+- Mention `@Openclaw` in #gremlins on Discord
+- Tag `@clockwork-labs-bot` on GitHub PRs/issues
+
+---
+
+*To add a new gremlin, document it here.*

.github/workflows/ci.yml

@@ -32,6 +32,7 @@ jobs:
     timeout-minutes: 120
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp
     steps:
       - name: Find Git ref
         env:
@@ -79,10 +80,36 @@ jobs:
         with:
           run_install: true
 
+      # Install emscripten for C++ module compilation tests.
+      - name: Install emscripten (Linux)
+        if: runner.os == 'Linux'
+        shell: bash
+        run: |
+          git clone https://github.com/emscripten-core/emsdk.git ~/emsdk
+          cd ~/emsdk
+          ./emsdk install 4.0.21
+          ./emsdk activate 4.0.21
+
+      - name: Install emscripten (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          git clone https://github.com/emscripten-core/emsdk.git $env:USERPROFILE\emsdk
+          cd $env:USERPROFILE\emsdk
+          .\emsdk install 4.0.21
+          .\emsdk activate 4.0.21
+
       - name: Install psql (Windows)
         if: runner.os == 'Windows'
-        run: choco install psql -y --no-progress
-        shell: powershell
+        shell: pwsh
+        run: |
+          # Fail properly if any individual command fails
+          $ErrorActionPreference = 'Stop'
+          $PSNativeCommandUseErrorActionPreference = $true
+          choco install psql -y --no-progress
+          # Check for existence, since `choco` doesn't seem to fail the step if it fails to install..
+          # See https://github.com/clockworklabs/SpacetimeDB/pull/4399 for more background.
+          Get-Command psql
 
       - name: Update dotnet workloads
         if: runner.os == 'Windows'
@@ -122,137 +149,34 @@ jobs:
       - name: Install cargo-nextest
         uses: taiki-e/install-action@nextest
 
-      - name: Run smoketests
-        # --test-threads=1 eliminates contention in the C# tests where they fight over bindings
-        # build artifacts.
-        # It also seemed to improve performance a fair amount (11m -> 6m)
-        run: cargo ci smoketests -- --test-threads=1
-
-  smoketests-python:
-    needs: [lints]
-    name: Smoketests (Python Legacy) (${{matrix.name}})
-    strategy:
-      matrix:
-        include:
-          - name: Linux
-            runner: spacetimedb-new-runner-2
-            smoketest_args: --docker
-          - name: Windows
-            runner: windows-latest
-            smoketest_args: --no-build-cli
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 120
-    env:
-      CARGO_TARGET_DIR: ${{ github.workspace }}/target
-    steps:
-      - name: Find Git ref
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        shell: bash
-        run: |
-          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
-          if test -n "${PR_NUMBER}"; then
-            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
-          else
-            GIT_REF="${{ github.ref }}"
-          fi
-          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
-      - name: Checkout sources
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ env.GIT_REF }}
-      - uses: dsherret/rust-toolchain-file@v1
-      - name: Set default rust toolchain
-        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: ${{ github.workspace }}
-          shared-key: spacetimedb
-          cache-on-failure: false
-          cache-all-crates: true
-          cache-workspace-crates: true
-          prefix-key: v1
-
-      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
-      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
-      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
-      # `cargo build --manifest-path` (which apparently build different dependency trees).
-      # However, we've been unable to fix it so... /shrug
-      - name: Check v8 outputs
+      # --test-threads=1 eliminates contention in the C# tests where they fight over bindings
+      # build artifacts.
+      # It also seemed to improve performance a fair amount (11m -> 6m)
+      - name: Run smoketests (Linux)
+        if: runner.os == 'Linux'
         shell: bash
         run: |
-          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
-          if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
-            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
-            cargo clean -p v8 || true
-            cargo build -p v8
+          if [ -f ~/emsdk/emsdk_env.sh ]; then
+            source ~/emsdk/emsdk_env.sh
           fi
+          cargo ci smoketests -- --test-threads=1
 
-      - uses: actions/setup-dotnet@v4
-        with:
-          global-json-file: global.json
-
-      - name: Override NuGet packages
-        shell: bash
-        run: |
-          dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime
-          dotnet pack -c Release crates/bindings-csharp/Runtime
-          cd sdks/csharp
-          ./tools~/write-nuget-config.sh ../..
-
-      # nodejs and pnpm are required for the typescript quickstart smoketest
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - uses: pnpm/action-setup@v4
-        with:
-          run_install: true
-
-      - name: Install psql (Windows)
+      # Due to Emscripten PATH issues this was separated to make sure OpenSSL still builds correctly
+      - name: Run smoketests (Windows)
         if: runner.os == 'Windows'
-        run: choco install psql -y --no-progress
-        shell: powershell
-
-      - name: Build crates
-        run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
-
-      - name: Build and start database (Linux)
-        if: runner.os == 'Linux'
+        shell: pwsh
         run: |
-          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
-          rm .dockerignore
-          docker compose -f .github/docker-compose.yml up -d
+          if (Test-Path "$env:USERPROFILE\emsdk\emsdk_env.ps1") {
+            & "$env:USERPROFILE\emsdk\emsdk_env.ps1" | Out-Null
+          }
+          cargo ci smoketests -- --test-threads=1          
 
-      - name: Build and start database (Windows)
-        if: runner.os == 'Windows'
+      - name: Check for changes
         run: |
-          # Fail properly if any individual command fails
-          $ErrorActionPreference = 'Stop'
-          $PSNativeCommandUseErrorActionPreference = $true
-
-          Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
-          cd modules
-          # the sdk-manifests on windows-latest are messed up, so we need to update them
-          dotnet workload config --update-mode manifests
-          dotnet workload update
-
-      - uses: actions/setup-python@v5
-        with: { python-version: "3.12" }
-        if: runner.os == 'Windows'
-
-      - name: Install python deps
-        run: python -m pip install -r smoketests/requirements.txt
-
-      - name: Run Python smoketests
-        # Note: clear_database and replication only work in private
-        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
-
-      - name: Stop containers (Linux)
-        if: always() && runner.os == 'Linux'
-        run: docker compose -f .github/docker-compose.yml down
+          tools/check-diff.sh crates/smoketests || {
+              echo 'Error: There is a diff in the smoketests directory.'
+              exit 1
+          }
 
   test:
     needs: [lints]
@@ -806,6 +730,12 @@ jobs:
           key: Unity-${{ github.head_ref }}
           restore-keys: Unity-
 
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
       - name: Run Unity tests
         uses: game-ci/unity-test-runner@v4
         with:
@@ -916,14 +846,15 @@ jobs:
               exit 1
           }
 
-      - name: Check client-api bindings are up to date
-        working-directory: sdks/csharp
-        run: |
-          bash tools~/gen-client-api.sh
-          "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/SpacetimeDB/ClientApi || {
-              echo 'Error: Client API bindings are dirty. Please run `sdks/csharp/tools~/gen-client-api.sh`.'
-              exit 1
-          }
+      # TODO: Re-enable this once csharp is using the v2 ws api.
+      # - name: Check client-api bindings are up to date
+      #   working-directory: sdks/csharp
+      #   run: |
+      #     bash tools~/gen-client-api.sh
+      #     "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/SpacetimeDB/ClientApi || {
+      #         echo 'Error: Client API bindings are dirty. Please run `sdks/csharp/tools~/gen-client-api.sh`.'
+      #         exit 1
+      #     }
 
       - name: Start SpacetimeDB
         run: |
@@ -1130,3 +1061,59 @@ jobs:
           fi
 
           echo "No Python smoketest changes detected."
+
+  smoketests_mod_rs_complete:
+    name: Check smoketests/mod.rs is complete
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+    steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Set default rust toolchain
+        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          cache-on-failure: false
+          cache-all-crates: true
+          cache-workspace-crates: true
+          prefix-key: v1
+
+      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
+      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
+      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
+      # `cargo build --manifest-path` (which apparently build different dependency trees).
+      # However, we've been unable to fix it so... /shrug
+      - name: Check v8 outputs
+        shell: bash
+        run: |
+          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
+          if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
+            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
+            cargo clean -p v8 || true
+            cargo build -p v8
+          fi
+
+      - name: Verify crates/smoketests/tests/smoketests/mod.rs lists all entries
+        run: |
+          cargo ci smoketests check-mod-list