First pass at reorganizing the docs and making them nice #15096

Workflow file for this run

	on:
	pull_request:
	push:
	branches:
	- master
	merge_group:
	workflow_dispatch:
	inputs:
	pr_number:
	description: 'Pull Request Number'
	required: false
	default: ''

	name: CI

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.event.inputs.pr_number \|\| format('sha-{0}', github.sha) }}
	cancel-in-progress: true

	jobs:
	docker_smoketests:
	name: Smoketests
	strategy:
	matrix:
	runner: [spacetimedb-new-runner, windows-latest]
	include:
	- runner: spacetimedb-new-runner
	smoketest_args: --docker
	container:
	image: localhost:5000/spacetimedb-ci:latest
	options: --privileged
	- runner: windows-latest
	smoketest_args: --no-build-cli
	container: null
	runs-on: ${{ matrix.runner }}
	container: ${{ matrix.container }}
	env:
	CARGO_TARGET_DIR: ${{ github.workspace }}/target
	steps:
	- name: Find Git ref
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	shell: bash
	run: \|
	PR_NUMBER="${{ github.event.inputs.pr_number \|\| null }}"
	if test -n "${PR_NUMBER}"; then
	GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
	else
	GIT_REF="${{ github.ref }}"
	fi
	echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
	- name: Checkout sources
	uses: actions/checkout@v4
	with:
	ref: ${{ env.GIT_REF }}
	- uses: dsherret/rust-toolchain-file@v1
	- name: Cache Rust dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces: ${{ github.workspace }}
	shared-key: spacetimedb
	cache-on-failure: true
	cache-all-crates: true
	cache-workspace-crates: true

	- uses: actions/setup-dotnet@v4
	with:
	global-json-file: global.json

	# nodejs and pnpm are required for the typescript quickstart smoketest
	- name: Set up Node.js
	uses: actions/setup-node@v4
	with:
	node-version: 18

	- uses: pnpm/action-setup@v4
	with:
	run_install: true

	- name: Install psql (Windows)
	if: runner.os == 'Windows'
	run: choco install psql -y --no-progress
	shell: powershell
	- name: Build crates
	run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
	- name: Start Docker daemon
	if: runner.os == 'Linux'
	run: /usr/local/bin/start-docker.sh

	- name: Build and start database (Linux)
	if: runner.os == 'Linux'
	run: \|
	# Our .dockerignore omits `target`, which our CI Dockerfile needs.
	rm .dockerignore
	docker compose -f .github/docker-compose.yml up -d
	- name: Build and start database (Windows)
	if: runner.os == 'Windows'
	run: \|
	# Fail properly if any individual command fails
	$ErrorActionPreference = 'Stop'
	$PSNativeCommandUseErrorActionPreference = $true

	Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
	cd modules
	# the sdk-manifests on windows-latest are messed up, so we need to update them
	dotnet workload config --update-mode manifests
	dotnet workload update
	- uses: actions/setup-python@v5
	with: { python-version: '3.12' }
	if: runner.os == 'Windows'
	- name: Install python deps
	run: python -m pip install -r smoketests/requirements.txt
	- name: Run smoketests
	# Note: clear_database and replication only work in private
	run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
	- name: Stop containers (Linux)
	if: always() && runner.os == 'Linux'
	run: docker compose -f .github/docker-compose.yml down

	test:
	name: Test Suite
	runs-on: spacetimedb-new-runner
	container:
	image: localhost:5000/spacetimedb-ci:latest
	options: >-
	--privileged
	env:
	CARGO_TARGET_DIR: ${{ github.workspace }}/target
	steps:
	- name: Find Git ref
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	PR_NUMBER="${{ github.event.inputs.pr_number \|\| null }}"
	if test -n "${PR_NUMBER}"; then
	GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
	else
	GIT_REF="${{ github.ref }}"
	fi
	echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"

	- name: Checkout sources
	uses: actions/checkout@v4
	with:
	ref: ${{ env.GIT_REF }}

	- uses: dsherret/rust-toolchain-file@v1
	- name: Cache Rust dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces: ${{ github.workspace }}
	shared-key: spacetimedb
	# Let the smoketests job save the cache since it builds the most things
	save-if: false

	- uses: actions/setup-dotnet@v3
	with:
	global-json-file: global.json

	- name: Set up Node.js
	uses: actions/setup-node@v4
	with:
	node-version: 18

	- uses: pnpm/action-setup@v4
	with:
	run_install: true

	- name: Create /stdb dir
	run: \|
	sudo mkdir /stdb
	sudo chmod 777 /stdb

	- name: Build typescript module sdk
	working-directory: crates/bindings-typescript
	run: pnpm build

	- name: Run cargo test
	#Note: Unreal tests will be run separately
	run: cargo test --all -- --skip unreal

	# The fallocate tests have been flakely when running in parallel
	- name: Run fallocate tests
	run: cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1

	- name: Check that the test outputs are up-to-date
	run: bash tools/check-diff.sh

	- name: Ensure C# autogen bindings are up-to-date
	run: \|
	cargo run -p spacetimedb-codegen --example regen-csharp-moduledef
	bash tools/check-diff.sh crates/bindings-csharp

	- name: C# bindings tests
	working-directory: crates/bindings-csharp
	run: dotnet test -warnaserror

	lints:
	name: Lints
	runs-on: spacetimedb-new-runner
	container:
	image: localhost:5000/spacetimedb-ci:latest
	options: >-
	--privileged
	env:
	CARGO_TARGET_DIR: ${{ github.workspace }}/target
	steps:
	- name: Checkout sources
	uses: actions/checkout@v3

	- uses: dsherret/rust-toolchain-file@v1
	- run: echo ::add-matcher::.github/workflows/rust_matcher.json

	- name: Cache Rust dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces: ${{ github.workspace }}
	shared-key: spacetimedb
	# Let the smoketests job save the cache since it builds the most things
	save-if: false

	- uses: actions/setup-dotnet@v3
	with:
	global-json-file: global.json

	- name: Run cargo fmt
	run: cargo fmt --all -- --check

	- name: Run cargo clippy
	run: cargo clippy --all --tests --benches -- -D warnings

	- name: Run C# formatting check
	working-directory: crates/bindings-csharp
	run: \|
	dotnet tool restore
	dotnet csharpier --check .

	- name: Run `cargo doc` for bindings crate
	# `bindings` is the only crate we care strongly about documenting,
	# since we link to its docs.rs from our website.
	# We won't pass `--no-deps`, though,
	# since we want everything reachable through it to also work.
	# This includes `sats` and `lib`.
	working-directory: crates/bindings
	env:
	# Make `cargo doc` exit with error on warnings, most notably broken links
	RUSTDOCFLAGS: '--deny warnings'
	run: \|
	cargo doc

	wasm_bindings:
	name: Build and test wasm bindings
	runs-on: spacetimedb-new-runner
	container:
	image: localhost:5000/spacetimedb-ci:latest
	options: >-
	--privileged
	env:
	CARGO_TARGET_DIR: ${{ github.workspace }}/target
	steps:
	- uses: actions/checkout@v3

	- uses: dsherret/rust-toolchain-file@v1
	- run: echo ::add-matcher::.github/workflows/rust_matcher.json

	- name: Cache Rust dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces: ${{ github.workspace }}
	shared-key: spacetimedb
	# Let the smoketests job save the cache since it builds the most things
	save-if: false

	- name: Run bindgen tests
	run: cargo test -p spacetimedb-codegen

	# Make sure the `Cargo.lock` file reflects the latest available versions.
	# This is what users would end up with on a fresh module, so we want to
	# catch any compile errors arising from a different transitive closure
	# of dependencies than what is in the workspace lock file.
	#
	# For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714
	- name: Update dependencies
	run: cargo update

	- name: Build module-test
	run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test


	publish_checks:
	name: Check that packages are publishable
	runs-on: spacetimedb-new-runner
	container:
	image: localhost:5000/spacetimedb-ci:latest
	options: >-
	--privileged
	permissions: read-all
	steps:
	- uses: actions/checkout@v3
	- name: Set up Python env
	run: \|
	test -d venv \|\| python3 -m venv venv
	venv/bin/pip3 install argparse toml
	- name: Run checks
	run: \|
	set -ueo pipefail
	FAILED=0
	ROOTS=(spacetimedb spacetimedb-sdk)
	CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}")
	for crate_dir in $CRATES; do
	if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then
	FAILED=$(( $FAILED + 1 ))
	fi
	done
	if [ $FAILED -gt 0 ]; then
	exit 1
	fi

	update:
	name: Test spacetimedb-update flow (${{ matrix.target }})
	permissions: read-all
	strategy:
	matrix:
	include:
	- runner: spacetimedb-new-runner
	target: x86_64-unknown-linux-gnu
	container:
	image: localhost:5000/spacetimedb-ci:latest
	options: --privileged
	- { target: aarch64-unknown-linux-gnu, runner: arm-runner }
	- { target: aarch64-apple-darwin, runner: macos-latest }
	- { target: x86_64-pc-windows-msvc, runner: windows-latest }
	runs-on: ${{ matrix.runner }}
	container: ${{ matrix.container }}
	steps:
	- name: Checkout
	uses: actions/checkout@v3

	- name: Install Rust
	uses: dsherret/rust-toolchain-file@v1

	- name: Install rust target
	run: rustup target add ${{ matrix.target }}

	- name: Install packages
	if: ${{ matrix.runner == 'arm-runner' }}
	shell: bash
	run: sudo apt install -y libssl-dev

	- name: Build spacetimedb-update
	run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update

	- name: Run self-install
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	shell: bash
	run: \|
	ROOT_DIR="$(mktemp -d)"
	# NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
	# My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by all users of that runner (I think this because it
	# happens very frequently on the `macos-runner`, but we haven't seen it on any others).
	cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
	"${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help

	unreal_engine_tests:
	name: Unreal Engine Tests
	# This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
	# a custom runner.
	runs-on: spacetimedb-new-runner
	# Disable the tests because they are very flaky at the moment.
	# TODO: Remove this line and re-enable the `if` line just below here.
	if: false
	# Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
	# if: ${{ github.event_name != 'pull_request' \|\| !github.event.pull_request.head.repo.fork }}
	container:
	image: ghcr.io/epicgames/unreal-engine:dev-5.6
	credentials:
	# Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and
	# it seems to be totally happy.
	# However, the token needs to be for a user that has access to the EpicGames org (see
	# https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6)
	username: ${{ github.actor }}
	password: ${{ secrets.GHCR_TOKEN }}
	# Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run
	# without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle.
	options: --user 0:0
	steps:
	# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back
	# commits though.
	# - name: Find Git ref
	# env:
	# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	# shell: bash
	# run: \|
	# PR_NUMBER="${{ github.event.inputs.pr_number \|\| null }}"
	# if test -n "${PR_NUMBER}"; then
	# GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
	# else
	# GIT_REF="${{ github.ref }}"
	# fi
	# echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
	- name: Checkout sources
	uses: actions/checkout@v4
	with:
	ref: ${{ env.GIT_REF }}
	- uses: dsherret/rust-toolchain-file@v1
	- name: Run Unreal Engine tests
	working-directory: sdks/unreal
	env:
	UE_ROOT_PATH: /home/ue4/UnrealEngine
	run: \|

	apt-get update
	apt-get install -y acl curl ca-certificates

	REPO="$GITHUB_WORKSPACE"
	# Let ue4 read/write the workspace & tool caches without changing ownership
	for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do
	[ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" \|\| true
	done

	# Rust tool caches live under the runner tool cache so they persist
	export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo"
	export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup"
	mkdir -p "$CARGO_HOME" "$RUSTUP_HOME"
	chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME"

	# Make sure the UE build script is executable (and parents traversable)
	UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}"
	chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" \|\| true
	chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" \|\| true

	# Run the build & tests as ue4 (who owns the UE tree)
	sudo -E -H -u ue4 env \
	HOME=/home/ue4 \
	XDG_CONFIG_HOME=/home/ue4/.config \
	CARGO_HOME="$CARGO_HOME" \
	RUSTUP_HOME="$RUSTUP_HOME" \
	PATH="$CARGO_HOME/bin:$PATH" \
	bash -lc '
	set -euxo pipefail
	# Install rustup for ue4 if needed (uses the shared caches)
	if ! command -v cargo >/dev/null 2>&1; then
	curl -sSf https://sh.rustup.rs \| sh -s -- -y
	fi
	rustup show >/dev/null
	git config --global --add safe.directory "$GITHUB_WORKSPACE" \|\| true

	cd "$GITHUB_WORKSPACE/sdks/unreal"
	cargo --version
	cargo test -- --test-threads=1
	'

	cli_docs:
	name: Check CLI docs
	permissions: read-all
	runs-on: spacetimedb-new-runner
	container:
	image: localhost:5000/spacetimedb-ci:latest
	options: >-
	--privileged
	env:
	CARGO_TARGET_DIR: ${{ github.workspace }}/target
	steps:
	- name: Find Git ref
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	shell: bash
	run: \|
	PR_NUMBER="${{ github.event.inputs.pr_number \|\| null }}"
	if test -n "${PR_NUMBER}"; then
	GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
	else
	GIT_REF="${{ github.ref }}"
	fi
	echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"

	- name: Checkout sources
	uses: actions/checkout@v4
	with:
	ref: ${{ env.GIT_REF }}

	- name: Set up Node.js
	uses: actions/setup-node@v4
	with:
	node-version: 18

	- uses: pnpm/action-setup@v4
	with:
	run_install: true

	- name: Get pnpm store directory
	shell: bash
	run: \|
	echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

	- uses: actions/cache@v4
	name: Setup pnpm cache
	with:
	path: ${{ env.STORE_PATH }}
	key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
	restore-keys: \|
	${{ runner.os }}-pnpm-store-

	- uses: dsherret/rust-toolchain-file@v1

	- name: Cache Rust dependencies
	uses: Swatinem/rust-cache@v2
	with:
	workspaces: ${{ github.workspace }}
	shared-key: spacetimedb
	# Let the smoketests job save the cache since it builds the most things
	save-if: false

	- name: Check for docs change
	run: \|
	cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md
	pnpm format
	git status
	if git diff --exit-code HEAD -- docs/docs/cli-reference.md; then
	echo "No docs changes detected"
	else
	echo "It looks like the CLI docs have changed:"
	exit 1
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

First pass at reorganizing the docs and making them nice #15096

Workflow file

First pass at reorganizing the docs and making them nice #15096

Uh oh!

Workflow file for this run