ci.yml

on:
  pull_request:
  push:
    branches:
      - master
  merge_group:
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'Pull Request Number'
        required: false
        default: ''

name: CI

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || format('sha-{0}', github.sha) }}
  cancel-in-progress: true

jobs:
  docker_smoketests:
    name: Smoketests
    strategy:
      matrix:
        runner: [spacetimedb-new-runner, windows-runner]
        include:
          - runner: spacetimedb-new-runner
            smoketest_args: --docker
            container:
              image: localhost:5000/spacetimedb-ci:latest
              options: --privileged
          - runner: windows-latest
            smoketest_args: --no-build-cli
            container: null
    runs-on: ${{ matrix.runner }}
    container: ${{ matrix.container }}
    
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}
      - uses: dsherret/rust-toolchain-file@v1
      - uses: actions/setup-dotnet@v4
        with:
          global-json-file: global.json

      # nodejs and pnpm are required for the typescript quickstart smoketest
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        with:
          run_install: true

      - name: Install psql (Windows)
        if: runner.os == 'Windows'
        run: choco install psql -y --no-progress
        shell: powershell
      - name: Start Docker daemon
        if: runner.os == 'Linux'
        run: /usr/local/bin/start-docker.sh

      - name: Build and start database (Linux)
        if: runner.os == 'Linux'
        run: docker compose up -d
      - name: Build and start database (Windows)
        if: runner.os == 'Windows'
        run: |
          # Fail properly if any individual command fails
          $ErrorActionPreference = 'Stop'
          $PSNativeCommandUseErrorActionPreference = $true

          cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
          Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
          cd modules
          # the sdk-manifests on windows-latest are messed up, so we need to update them
          dotnet workload config --update-mode manifests
          dotnet workload update
      - uses: actions/setup-python@v5
        with: { python-version: '3.12' }
        if: runner.os == 'Windows'
      - name: Install python deps
        run: python -m pip install psycopg2-binary xmltodict
      - name: Run smoketests
        # Note: clear_database and replication only work in private
        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
      - name: Stop containers (Linux)
        if: always() && runner.os == 'Linux'
        run: docker compose down

  test:
    name: Test Suite
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}

      - uses: dsherret/rust-toolchain-file@v1

      - uses: actions/setup-dotnet@v3
        with:
          global-json-file: global.json

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        with:
          run_install: true

      - name: Create /stdb dir
        run: |
          sudo mkdir /stdb
          sudo chmod 777 /stdb

      - name: Build typescript module sdk
        working-directory: crates/bindings-typescript
        run: pnpm build

      - name: Run cargo test
        #Note: Unreal tests will be run separately
        run: cargo test --all -- --skip unreal

        # The fallocate tests have been flakely when running in parallel
      - name: Run fallocate tests
        run: cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1

      - name: Check that the test outputs are up-to-date
        run: bash tools/check-diff.sh

      - name: Ensure C# autogen bindings are up-to-date
        run: |
          cargo run -p spacetimedb-codegen --example regen-csharp-moduledef
          bash tools/check-diff.sh crates/bindings-csharp

      - name: C# bindings tests
        working-directory: crates/bindings-csharp
        run: dotnet test -warnaserror

  lints:
    name: Lints
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - uses: dsherret/rust-toolchain-file@v1
      - run: echo ::add-matcher::.github/workflows/rust_matcher.json

      - uses: actions/setup-dotnet@v3
        with:
          global-json-file: global.json

      - name: Run cargo fmt
        run: cargo fmt --all -- --check

      - name: Run cargo clippy
        run: cargo clippy --all --tests --benches -- -D warnings

      - name: Run C# formatting check
        working-directory: crates/bindings-csharp
        run: |
          dotnet tool restore
          dotnet csharpier --check .

      - name: Run `cargo doc` for bindings crate
        # `bindings` is the only crate we care strongly about documenting,
        # since we link to its docs.rs from our website.
        # We won't pass `--no-deps`, though,
        # since we want everything reachable through it to also work.
        # This includes `sats` and `lib`.
        working-directory: crates/bindings
        env:
          # Make `cargo doc` exit with error on warnings, most notably broken links
          RUSTDOCFLAGS: '--deny warnings'
        run: |
          cargo doc

  wasm_bindings:
    name: Build and test wasm bindings
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    steps:
      - uses: actions/checkout@v3

      - uses: dsherret/rust-toolchain-file@v1
      - run: echo ::add-matcher::.github/workflows/rust_matcher.json

      - name: Run bindgen tests
        run: cargo test -p spacetimedb-codegen

      # Make sure the `Cargo.lock` file reflects the latest available versions.
      # This is what users would end up with on a fresh module, so we want to
      # catch any compile errors arising from a different transitive closure
      # of dependencies than what is in the workspace lock file.
      #
      # For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714
      - name: Update dependencies
        run: cargo update

      - name: Build module-test
        run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test


  publish_checks:
    name: Check that packages are publishable
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    permissions: read-all
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python env
        run: |
          test -d venv || python3 -m venv venv
          venv/bin/pip3 install argparse toml
      - name: Run checks
        run: |
          set -ueo pipefail
          FAILED=0
          ROOTS=(spacetimedb spacetimedb-sdk)
          CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}")
          for crate_dir in $CRATES; do
            if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then
              FAILED=$(( $FAILED + 1 ))
            fi
          done
          if [ $FAILED -gt 0 ]; then
            exit 1
          fi

  update:
    name: Test spacetimedb-update flow
    permissions: read-all
    strategy:
      matrix:
        include:
          - { target: x86_64-unknown-linux-gnu, runner: spacetimedb-runner }
          - { target: aarch64-unknown-linux-gnu, runner: arm-runner }
          - { target: aarch64-apple-darwin, runner: macos-latest }
          - { target: x86_64-pc-windows-msvc, runner: windows-latest }
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install Rust
        uses: dsherret/rust-toolchain-file@v1

      - name: Install rust target
        run: rustup target add ${{ matrix.target }}

      - name: Install packages
        if: ${{ matrix.runner == 'arm-runner' }}
        shell: bash
        run: sudo apt install -y libssl-dev

      - name: Build spacetimedb-update
        run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update

      - name: Run self-install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          ROOT_DIR="$(mktemp -d)"
          # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
          # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
          # happens very frequently on the `macos-runner`, but we haven't seen it on any others).
          cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
          "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help

  unreal_engine_tests:
    name: Unreal Engine Tests
    # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
    # a custom runner.
    runs-on: spacetimedb-new-runner
    # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
    if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
    container:
      image: ghcr.io/epicgames/unreal-engine:dev-5.6
      credentials:
        # Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and
        # it seems to be totally happy.
        # However, the token needs to be for a user that has access to the EpicGames org (see
        # https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6)
        username: ${{ github.actor }}
        password: ${{ secrets.GHCR_TOKEN }}
      # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run
      # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle.
      options: --user 0:0
    steps:
# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back
# commits though.
#      - name: Find Git ref
#        env:
#          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
#        shell: bash
#        run: |
#          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
#          if test -n "${PR_NUMBER}"; then
#            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
#          else
#            GIT_REF="${{ github.ref }}"
#          fi
#          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}
      - uses: dsherret/rust-toolchain-file@v1
      - name: Run Unreal Engine tests
        working-directory: sdks/unreal
        env:
          UE_ROOT_PATH: /home/ue4/UnrealEngine
        run: |

          apt-get update
          apt-get install -y acl curl ca-certificates

          REPO="$GITHUB_WORKSPACE"
          # Let ue4 read/write the workspace & tool caches without changing ownership
          for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do
            [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true
          done

          # Rust tool caches live under the runner tool cache so they persist
          export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo"
          export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup"
          mkdir -p "$CARGO_HOME" "$RUSTUP_HOME"
          chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME"

          # Make sure the UE build script is executable (and parents traversable)
          UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}"
          chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true
          chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true

          # Run the build & tests as ue4 (who owns the UE tree)
          sudo -E -H -u ue4 env \
            HOME=/home/ue4 \
            XDG_CONFIG_HOME=/home/ue4/.config \
            CARGO_HOME="$CARGO_HOME" \
            RUSTUP_HOME="$RUSTUP_HOME" \
            PATH="$CARGO_HOME/bin:$PATH" \
            bash -lc '
              set -euxo pipefail
              # Install rustup for ue4 if needed (uses the shared caches)
              if ! command -v cargo >/dev/null 2>&1; then
                curl -sSf https://sh.rustup.rs | sh -s -- -y
              fi
              rustup show >/dev/null
              git config --global --add safe.directory "$GITHUB_WORKSPACE" || true

              cd "$GITHUB_WORKSPACE/sdks/unreal"
              cargo --version
              cargo test -- --test-threads=1
            '

  cli_docs:
    name: Check CLI docs
    permissions: read-all
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        with:
          run_install: true

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - uses: actions/cache@v4
        name: Setup pnpm cache
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-

      - uses: dsherret/rust-toolchain-file@v1

      - name: Check for docs change
        run: |
          cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md
          pnpm format
          git status
          if git diff --exit-code HEAD; then
            echo "No docs changes detected"
          else
            echo "It looks like the CLI docs have changed:"
            exit 1
          fi