add initial cargo ci

.cargo/config.toml

@@ -3,6 +3,7 @@ rustflags = ["--cfg", "tokio_unstable"]
 
 [alias]
 bump-versions = "run -p upgrade-version --"
+ci = "run -p ci --"
 
 [target.x86_64-pc-windows-msvc]
 # Use a different linker. Otherwise, the build fails with some obscure linker error that

.github/workflows/ci.yml

@@ -112,7 +112,7 @@
         run: python -m pip install -r smoketests/requirements.txt
       - name: Run smoketests
         # Note: clear_database and replication only work in private
-        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
+        run: cargo ci smoketests -- ${{ matrix.smoketest_args }} -x clear_database replication teams
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
         run: docker compose -f .github/docker-compose.yml down
@@ -166,34 +166,12 @@
         with:
           run_install: true
 
-      - name: Create /stdb dir
-        run: |
-          sudo mkdir /stdb
-          sudo chmod 777 /stdb
-
       - name: Build typescript module sdk
         working-directory: crates/bindings-typescript
         run: pnpm build
 
-      - name: Run cargo test
-        #Note: Unreal tests will be run separately
-        run: cargo test --all -- --skip unreal
-
-        # The fallocate tests have been flakely when running in parallel
-      - name: Run fallocate tests
-        run: cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1
-
-      - name: Check that the test outputs are up-to-date
-        run: bash tools/check-diff.sh
-
-      - name: Ensure C# autogen bindings are up-to-date
-        run: |
-          cargo run -p spacetimedb-codegen --example regen-csharp-moduledef
-          bash tools/check-diff.sh crates/bindings-csharp
-
-      - name: C# bindings tests
-        working-directory: crates/bindings-csharp
-        run: dotnet test -warnaserror
+      - name: Run tests
+        run: cargo ci test
 
   lints:
     name: Lints
@@ -223,30 +201,8 @@
         with:
           global-json-file: global.json
 
-      - name: Run cargo fmt
-        run: cargo fmt --all -- --check
-
-      - name: Run cargo clippy
-        run: cargo clippy --all --tests --benches -- -D warnings
-
-      - name: Run C# formatting check
-        working-directory: crates/bindings-csharp
-        run: |
-          dotnet tool restore
-          dotnet csharpier --check .
-
-      - name: Run `cargo doc` for bindings crate
-        # `bindings` is the only crate we care strongly about documenting,
-        # since we link to its docs.rs from our website.
-        # We won't pass `--no-deps`, though,
-        # since we want everything reachable through it to also work.
-        # This includes `sats` and `lib`.
-        working-directory: crates/bindings
-        env:
-          # Make `cargo doc` exit with error on warnings, most notably broken links
-          RUSTDOCFLAGS: '--deny warnings'
-        run: |
-          cargo doc
+      - name: Run ci lint
+        run: cargo ci lint
 
   wasm_bindings:
     name: Build and test wasm bindings
@@ -272,20 +228,7 @@
           save-if: false
 
       - name: Run bindgen tests
-        run: cargo test -p spacetimedb-codegen
-
-      # Make sure the `Cargo.lock` file reflects the latest available versions.
-      # This is what users would end up with on a fresh module, so we want to
-      # catch any compile errors arising from a different transitive closure
-      # of dependencies than what is in the workspace lock file.
-      #
-      # For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714
-      - name: Update dependencies
-        run: cargo update
-
-      - name: Build module-test
-        run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test
-
+        run: cargo ci wasm-bindings
 
   publish_checks:
     name: Check that packages are publishable
@@ -343,19 +286,9 @@
         run: sudo apt install -y libssl-dev
 
       - name: Build spacetimedb-update
-        run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update
-
-      - name: Run self-install
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        shell: bash
-        run: |
-          ROOT_DIR="$(mktemp -d)"
-          # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
-          # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
-          # happens very frequently on the `macos-runner`, but we haven't seen it on any others).
-          cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
-          "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help
+        run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth
 
   unreal_engine_tests:
     name: Unreal Engine Tests
@@ -446,7 +379,34 @@
               cargo test -- --test-threads=1
             '
 
+  ci_command_docs:
+    name: Check CI command docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+
+      - uses: dsherret/rust-toolchain-file@v1
+
+      - name: Check for docs change
+        run: cargo ci self-docs --check
+
   cli_docs:
     name: Check CLI docs
     permissions: read-all
     runs-on: spacetimedb-new-runner
@@ -509,12 +469,4 @@
 
       - name: Check for docs change
         run: |
-          cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md
-          pnpm format
-          git status
-          if git diff --exit-code HEAD; then
-            echo "No docs changes detected"
-          else
-            echo "It looks like the CLI docs have changed:"
-            exit 1
-          fi
+          cargo ci cli-docs

Cargo.toml

@@ -48,6 +48,7 @@ members = [
   "sdks/rust/tests/test-counter",
   "sdks/rust/tests/connect_disconnect_client",
   "sdks/rust/tests/procedure-client",
+  "tools/ci",
   "tools/upgrade-version",
   "tools/license-check",
   "tools/replace-spacetimedb",

tools/ci/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "ci"
+version = "0.1.0"
+edition.workspace = true
+
+[dependencies]
+log.workspace = true
+anyhow.workspace = true
+chrono = { workspace = true, features=["clock"] }
+clap.workspace = true
+regex.workspace = true
+duct.workspace = true

tools/ci/README.md

@@ -0,0 +1,148 @@
+# SpacetimeDB's cargo ci
+
+## Overview
+
+This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options.
+
+## `cargo ci`
+
+SpacetimeDB CI tasks
+
+This tool provides several subcommands for automating CI workflows in SpacetimeDB.
+
+It may be invoked via `cargo ci <subcommand>`, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally
+
+**Usage:**
+```bash
+Usage: cargo ci [OPTIONS] [COMMAND]
+```
+
+**Options:**
+
+- `--skip`: Skip specified subcommands when running all
+
+When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`.
+
+- `--help`: Print help (see a summary with '-h')
+
+### `test`
+
+Runs tests
+
+Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state.
+
+**Usage:**
+```bash
+Usage: test
+```
+
+**Options:**
+
+- `--help`: Print help (see a summary with '-h')
+
+### `lint`
+
+Lints the codebase
+
+Runs rustfmt, clippy, csharpier and generates rust docs to ensure there are no warnings.
+
+**Usage:**
+```bash
+Usage: lint
+```
+
+**Options:**
+
+- `--help`: Print help (see a summary with '-h')
+
+### `wasm-bindings`
+
+Tests Wasm bindings
+
+Runs tests for the codegen crate and builds a test module with the wasm bindings.
+
+**Usage:**
+```bash
+Usage: wasm-bindings
+```
+
+**Options:**
+
+- `--help`: Print help (see a summary with '-h')
+
+### `smoketests`
+
+Runs smoketests
+
+Executes the smoketests suite with some default exclusions.
+
+**Usage:**
+```bash
+Usage: smoketests [ARGS]...
+```
+
+**Options:**
+
+- `args`: Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`
+- `--help`: Print help (see a summary with '-h')
+
+### `update-flow`
+
+Tests the update flow
+
+Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory.
+
+**Usage:**
+```bash
+Usage: update-flow [OPTIONS]
+```
+
+**Options:**
+
+- `--target`: Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms.
+- `--github-token-auth`: Whether to enable github token authentication feature when building the update binary. By default this is disabled.
+- `--help`: Print help (see a summary with '-h')
+
+### `cli-docs`
+
+**Usage:**
+```bash
+Usage: cli-docs [OPTIONS]
+```
+
+**Options:**
+
+- `--spacetime-path`: specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)
+- `--help`: Print help (see a summary with '-h')
+
+### `self-docs`
+
+**Usage:**
+```bash
+Usage: self-docs [OPTIONS]
+```
+
+**Options:**
+
+- `--check`: Only check for changes, do not generate the docs
+- `--help`: Print help (see a summary with '-h')
+
+### `help`
+
+**Usage:**
+```bash
+Usage: help [COMMAND]...
+```
+
+**Options:**
+
+- `subcommand`: 
+
+
+---
+
+This document is auto-generated by running:
+
+```bash
+cargo ci self-docs
+```