Merge branch 'master' into bfops/pnpm-install-instruction

Signed-off-by: Zeke Foppa <196249+bfops@users.noreply.github.com>

Author: bfops

.cargo/config.toml

@@ -3,3 +3,13 @@ rustflags = ["--cfg", "tokio_unstable"]
 
 [alias]
 bump-versions = "run -p upgrade-version --"
+
+[target.x86_64-pc-windows-msvc]
+# Use a different linker. Otherwise, the build fails with some obscure linker error that
+# seems to be a result of us producing a massive PDB file.
+# I (@bfops) tried a variety of other link options besides switching linkers, but this
+# seems to be the only thing that worked.
+linker = "lld-link"
+# Without this, the linker complains that libc functions are undefined -
+# it probably signals to rustc and lld-link that libucrt should be included.
+rustflags = ["-Ctarget-feature=+crt-static"]

.env.sample

@@ -0,0 +1,2 @@
+# Directory environment using https://direnv.net/ and https://github.com/nix-community/nix-direnv.
+use flake

.envrc

@@ -13,16 +13,28 @@ on:
 
 name: CI
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || format('sha-{0}', github.sha) }}
+  cancel-in-progress: true
+
 jobs:
   docker_smoketests:
     name: Smoketests
     strategy:
       matrix:
+        runner: [spacetimedb-new-runner, windows-latest]
         include:
-          - { runner: spacetimedb-runner, smoketest_args: --docker }
-          - { runner: windows-latest, smoketest_args: --no-build-cli }
-        runner: [ spacetimedb-runner, windows-latest ]
+          - runner: spacetimedb-new-runner
+            smoketest_args: --docker
+            container:
+              image: localhost:5000/spacetimedb-ci:latest
+              options: --privileged
+          - runner: windows-latest
+            smoketest_args: --no-build-cli
+            container: null
     runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}
+    
     steps:
       - name: Find Git ref
         env:
@@ -44,10 +56,25 @@ jobs:
       - uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
+
+      # nodejs and pnpm are required for the typescript quickstart smoketest
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: true
+
       - name: Install psql (Windows)
         if: runner.os == 'Windows'
         run: choco install psql -y --no-progress
         shell: powershell
+      - name: Start Docker daemon
+        if: runner.os == 'Linux'
+        run: /usr/local/bin/start-docker.sh
+
       - name: Build and start database (Linux)
         if: runner.os == 'Linux'
         run: docker compose up -d
@@ -58,12 +85,6 @@ jobs:
           $ErrorActionPreference = 'Stop'
           $PSNativeCommandUseErrorActionPreference = $true
 
-          # Use a different linker. Otherwise, the build fails with some obscure linker error that
-          # seems to be a result of us producing a massive PDB file.
-          # I (@bfops) tried a variety of other link options besides switching linkers, but this
-          # seems to be the only thing that worked.
-          $env:RUSTFLAGS='-Clinker=lld-link'
-
           cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
           Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
           cd modules
@@ -73,18 +94,22 @@ jobs:
       - uses: actions/setup-python@v5
         with: { python-version: '3.12' }
         if: runner.os == 'Windows'
-      - name: Install psycopg2
-        run: python -m pip install psycopg2-binary
+      - name: Install python deps
+        run: python -m pip install psycopg2-binary xmltodict
       - name: Run smoketests
         # Note: clear_database and replication only work in private
-        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication
+        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
         run: docker compose down
 
   test:
     name: Test Suite
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - name: Find Git ref
         env:
@@ -109,15 +134,32 @@ jobs:
         with:
           global-json-file: global.json
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: true
+
       - name: Create /stdb dir
         run: |
           sudo mkdir /stdb
           sudo chmod 777 /stdb
 
+      - name: Build typescript module sdk
+        working-directory: crates/bindings-typescript
+        run: pnpm build
+
       - name: Run cargo test
         #Note: Unreal tests will be run separately
         run: cargo test --all -- --skip unreal
 
+        # The fallocate tests have been flakely when running in parallel
+      - name: Run fallocate tests
+        run: cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1
+
       - name: Check that the test outputs are up-to-date
         run: bash tools/check-diff.sh
 
@@ -132,7 +174,11 @@ jobs:
 
   lints:
     name: Lints
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -171,7 +217,11 @@ jobs:
 
   wasm_bindings:
     name: Build and test wasm bindings
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - uses: actions/checkout@v3
 
@@ -196,7 +246,11 @@ jobs:
 
   publish_checks:
     name: Check that packages are publishable
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     permissions: read-all
     steps:
       - uses: actions/checkout@v3
@@ -206,11 +260,12 @@ jobs:
           venv/bin/pip3 install argparse toml
       - name: Run checks
         run: |
+          set -ueo pipefail
           FAILED=0
-          # This definition of ROOTS and invocation of find-publish-list.py is copied from publish-crates.sh
-          ROOTS=(bindings sdk cli standalone)
-          for crate in $(venv/bin/python3 tools/find-publish-list.py --recursive --quiet "${ROOTS[@]}"); do
-            if ! venv/bin/python3 tools/crate-publish-checks.py "crates/$crate"; then
+          ROOTS=(spacetimedb spacetimedb-sdk)
+          CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}")
+          for crate_dir in $CRATES; do
+            if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then
               FAILED=$(( $FAILED + 1 ))
             fi
           done
@@ -263,7 +318,9 @@ jobs:
     name: Unreal Engine Tests
     # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
     # a custom runner.
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
+    if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
     container:
       image: ghcr.io/epicgames/unreal-engine:dev-5.6
       credentials:
@@ -340,13 +397,17 @@ jobs:
 
               cd "$GITHUB_WORKSPACE/sdks/unreal"
               cargo --version
-              cargo test
+              cargo test -- --test-threads=1
             '
 
   cli_docs:
     name: Check CLI docs
     permissions: read-all
-    runs-on: ubuntu-latest
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
     steps:
       - name: Find Git ref
         env:

.github/workflows/ci.yml

@@ -6,9 +6,18 @@ on:
       - master
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || format('sha-{0}', github.sha) }}
+  cancel-in-progress: true
+
 jobs:
   unity-testsuite:
-    runs-on: spacetimedb-runner
+    runs-on: spacetimedb-new-runner
+    container:
+      image: localhost:5000/spacetimedb-ci:latest
+      options: >-
+        --privileged
+        --cgroupns=host
     # Cancel any previous testsuites running on the same PR and/or ref.
     concurrency:
       group: unity-test-${{ github.event.pull_request.number || github.ref }}
@@ -77,7 +86,7 @@ jobs:
           cargo install --force --path crates/cli --locked --message-format=short
           cargo install --force --path crates/standalone --locked --message-format=short
           # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
-          ln -sf $HOME/.cargo/bin/spacetimedb-cli $HOME/.cargo/bin/spacetime
+          ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
         env:
           # Share the target directory with our local project to avoid rebuilding same SpacetimeDB crates twice.
           CARGO_TARGET_DIR: demo/Blackholio/server-rust/target
@@ -153,6 +162,9 @@ jobs:
           key: Unity-${{ github.head_ref }}
           restore-keys: Unity-
 
+        # We need this to support "Docker in Docker"
+      - name: Start Docker daemon
+        run: /usr/local/bin/start-docker.sh
       - name: Run Unity tests
         uses: game-ci/unity-test-runner@v4
         with:
@@ -167,4 +179,4 @@ jobs:
           UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
         # Skip if this is an external contribution.
         # The license secrets will be empty, so the step would fail anyway.
-        if: ${{ !github.event.pull_request.head.repo.fork }}
+        if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}

.github/workflows/csharp-test.yml

@@ -10,7 +10,6 @@ jobs:
     if: github.event.pull_request.merged == true &&
       github.event.pull_request.base.ref == 'master'
     env:
-      CHECK_NAME: Internal Tests
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Set up GitHub CLI
@@ -21,34 +20,51 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y gh
 
-      - name: Fetch Check Run Results
-        run: |
-          RESULT="$(gh pr checks "${{github.event.pull_request.html_url}}" --json 'name,state' |
-            jq -r ".[] | select(.name==\"${CHECK_NAME}\").state")"
-
-          if [ -z "$RESULT" ]; then
-            RESULT="The check did not run!"
-          fi
-
-          echo "CHECK_RESULT=${RESULT}" >> $GITHUB_ENV
-
       - name: Send Discord notification
         env:
           DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
           PR_TITLE: ${{ github.event.pull_request.title }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
           PR_URL: ${{ github.event.pull_request.html_url }}
-          CHECK_RESULT: ${{ env.CHECK_RESULT }}
           MENTION_ON_FAILURE: ${{ secrets.DEV_OPS_ROLE_ID }}
+          DISCORD_USER_MAP: ${{ secrets.DISCORD_USER_MAP }}
         run: |
-          message="PR merged: [(#${PR_NUMBER}) ${PR_TITLE}](${PR_URL})"
-          message+=$'\n'
-          message+="${CHECK_NAME} result: ${CHECK_RESULT}"
+          message="PR merged: [(#${PR_NUMBER}) ${PR_TITLE}](<${PR_URL}>)"
           # Note that anything besides success is treated as a failure (e.g. if the check did not run at all, or if it is still pending).
-          if [[ "${CHECK_RESULT}" != "SUCCESS" ]]; then
+          FAILED_CHECKS="$(
+            gh pr checks "${{github.event.pull_request.html_url}}" \
+              --json 'workflow,state,name' |
+              jq '.[]
+                | select(.workflow != "Discord notifications")
+                | select(.state != "SUCCESS" and .state != "NEUTRAL")
+              ' |
+              jq -r '"\(.workflow) / \(.name): \(.state)"'
+          )"
+
+          # Lookup PR author's Discord ID from the provided JSON map (if any)
+          author_discord_id="$(
+            jq -r \
+              --arg u "${{ github.event.pull_request.user.login }}" \
+              'try .[$u] catch empty' \
+              <<<"${DISCORD_USER_MAP}"
+          )"
+          if [ -z "${author_discord_id}" ]; then
+            echo "Warning: PR author not found not found in USER_LOOKUP_JSON"
+          fi
+
+          message+=$'\n'
+          if [[ -z "${FAILED_CHECKS}" ]]; then
+            message+='All checks passed.'
+          else
+            message+="${FAILED_CHECKS}"
+            message+=$'\n'
             # This uses special Discord syntax for pinging a particular role.
             # Note the '&' - this is the difference between pinging a *role* and pinging a *person*.
-            message+=" (cc <@&${MENTION_ON_FAILURE}>)"
+            message+=" (cc <@&${MENTION_ON_FAILURE}>"
+            if [[ -n "${author_discord_id}" ]]; then
+              message+=" <@${author_discord_id}>"
+            fi
+            message+=")"
           fi
           # Use `jq` to construct the json data blob in the format required by the webhook.
           data="$(jq --null-input --arg msg "$message" '.content=$msg')"