ci.yml

name: CI

on:
    pull_request:
        types: [opened, synchronize, reopened]
    push:
        branches: [main]
    workflow_dispatch:

permissions:
    contents: read

jobs:
    test-pinned:
        # Matrix Xcode coverage to catch SE-0482 artifactbundle resolution
        # regressions in Xcode's build system. Xcode 26.3 cannot resolve the
        # bundle's modulemap (issue #33) and is omitted; Apple fixed the
        # underlying bug in 26.4. Xcode 26.4+ requires macOS 26 (Tahoe), so
        # the matrix runs on macos-26.
        name: Test (pinned artifactbundle, Xcode ${{ matrix.xcode }})
        strategy:
            fail-fast: false
            matrix:
                include:
                    - os: macos-26
                      xcode: "26.4"
                    - os: macos-26
                      xcode: "26.5"
        runs-on: ${{ matrix.os }}
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
            - uses: ./.github/actions/select-xcode
              with:
                  version: ${{ matrix.xcode }}
            # Run both build systems: regressions can appear in one without the other.
            # TEST_RUNNER_ prefix is required for xcodebuild to forward the env var
            # to the xctest subprocess that actually emits the framework noise.
            - name: xcodebuild test
              env:
                  TEST_RUNNER_OS_ACTIVITY_MODE: disable
              run: |
                  set -euo pipefail
                  xcodebuild test \
                    -scheme swift-tokenizers \
                    -destination 'platform=macOS,arch=arm64' \
                    | xcbeautify --renderer github-actions

            - name: swift test
              run: swift test

    test-from-source:
        name: Test (Rust built from source)
        runs-on: macos-latest
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

            - uses: ./.github/actions/select-xcode

            - name: Read Rust toolchain version
              id: rust-toolchain
              run: |
                  version=$(awk -F'"' '/^channel/ {print $2; exit}' rust-toolchain.toml)
                  echo "version=$version" >> "$GITHUB_OUTPUT"

            - name: Install Rust toolchain
              uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
              with:
                  toolchain: ${{ steps.rust-toolchain.outputs.version }}
                  targets: aarch64-apple-darwin,x86_64-apple-darwin,aarch64-apple-ios,aarch64-apple-ios-sim,x86_64-apple-ios

            - name: Build Apple Rust slices
              run: bash scripts/rust/build/build-rust-apple-slices.sh

            - name: Assemble Apple-only artifactbundle
              # The Linux slices are built in a separate release-pipeline job.
              # For the from-source PR check on macOS we assemble whatever is
              # present (Apple slices only) — SwiftPM picks the host's variant
              # at link time, so an Apple-only bundle is sufficient here.
              run: bash scripts/rust/build/assemble-artifactbundle.sh ci-from-source

            - name: Run tests against locally built artifactbundle
              env:
                  TOKENIZERS_RUST_LOCAL_ARTIFACTBUNDLE_PATH: rust/target/artifactbundle/TokenizersRust.artifactbundle
              run: |
                  swift package clean
                  swift test

    test-linux:
        # Build the Rust crate for the host Linux target, assemble a
        # Linux-only artifactbundle, and run the offline test subset
        # against it. Catches PRs that touch `Sources/Tokenizers/`,
        # the UniFFI wrapper, or the Rust crate before merge rather
        # than at release time.
        #
        # The job filters to AlgorithmTests + UniffiSpikeTests because
        # the other test files go through `HubClient` from `swift-hf-api`,
        # which triggers a deterministic `libcurl.Easy Code=43`
        # (`CURLE_BAD_FUNCTION_ARGUMENT`) crash in
        # `_HTTPURLProtocol.configureEasyHandle` on Swift 6.2.x / 6.3.x +
        # Linux when curl reuses a handle across an HTTP redirect. The
        # crash is upstream FoundationNetworking, not anything in this
        # repo (which does not use URLSession at all).
        #
        #   Issue: https://github.com/swiftlang/swift-corelibs-foundation/issues/5445
        #   Fix:   https://github.com/swiftlang/swift-corelibs-foundation/pull/5448
        #          (merged 2026-04-08; drops the `preferredReceiveBufferSize`
        #          call that libcurl 8.5.0 rejects on a reused handle)
        #
        # The fix is on `main` and `release/6.4.x` only — not backported
        # to 6.2.x or 6.3.x — so the first user-visible Swift release
        # containing it is 6.4.
        #
        # Drop the `--filter` flags in the test step once *either*
        # condition holds:
        #   1. The `container:` is bumped to `swift:6.4` or later, so
        #      the FoundationNetworking fix is on the runtime stack.
        #   2. `swift-hf-api` migrates its `Hub/` subsystem to wrap
        #      the Rust `hf-hub` crate, so `HubClient` no longer goes
        #      through FoundationNetworking on Linux at all.
        name: Test (Linux, offline subset)
        runs-on: ubuntu-latest
        container: swift:6.2.3
        defaults:
            run:
                # Inside the swift container the runner falls back to /bin/sh,
                # which rejects `set -o pipefail`. Force bash so inline scripts
                # behave the same as on a regular Linux runner.
                shell: bash
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

            - name: Read Rust toolchain version
              id: rust-toolchain
              run: |
                  version=$(awk -F'"' '/^channel/ {print $2; exit}' rust-toolchain.toml)
                  echo "version=$version" >> "$GITHUB_OUTPUT"

            - name: Install rustup and the host Linux target
              # The container is a Swift toolchain image without rustup.
              # `dtolnay/rust-toolchain` does not work inside containers, so
              # bootstrap rustup directly.
              env:
                  RUST_VERSION: ${{ steps.rust-toolchain.outputs.version }}
              run: |
                  set -euo pipefail
                  apt-get update -qq
                  apt-get install -y --no-install-recommends \
                    curl ca-certificates build-essential pkg-config python3 zip
                  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
                    | sh -s -- -y --default-toolchain "${RUST_VERSION}" --profile minimal
                  echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"

            - name: Build host Linux Rust slice
              env:
                  TOKENIZERS_LINUX_TARGETS: x86_64-unknown-linux-gnu
              run: bash scripts/rust/build/build-rust-linux-archives.sh

            - name: Assemble Linux-only artifactbundle
              run: bash scripts/rust/build/assemble-artifactbundle.sh ci-linux-from-source

            - name: Run offline-only swift test against the locally assembled bundle
              env:
                  TOKENIZERS_RUST_LOCAL_ARTIFACTBUNDLE_PATH: rust/target/artifactbundle/TokenizersRust.artifactbundle
              run: |
                  swift package clean
                  # TODO: drop --filter flags once Swift 6.4 ships in
                  # the container or swift-hf-api migrates Hub/ to the
                  # Rust hf-hub crate. The job-level comment above has
                  # the full rationale and links.
                  swift test --filter Algorithm --filter UniffiSpike

    wrapper-drift-changes:
        # Detects whether the PR touched any input that influences the generated
        # UniFFI wrapper, so the (expensive) `wrapper-drift` job below can be
        # skipped on PRs that only touch unrelated paths.
        name: Detect wrapper-drift inputs
        runs-on: ubuntu-latest
        outputs:
            should-run: ${{ steps.filter.outputs.relevant }}
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

            - id: filter
              uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3
              with:
                  filters: |
                      relevant:
                          - 'rust/**'
                          - 'rust-toolchain.toml'
                          - 'Sources/TokenizersFFI/Generated/**'
                          - 'scripts/rust/**'
                          - '.github/workflows/ci.yml'

    wrapper-drift:
        # Confirms the committed UniFFI wrapper at
        # `Sources/TokenizersFFI/Generated/TokenizersFFI.swift` still matches what
        # the bindgen would emit for the current Rust source. The check rebuilds
        # only the host static library and the bindgen binary (no per-target
        # matrix, no artifactbundle assembly), but is still gated by
        # `wrapper-drift-changes` on PRs that don't touch the relevant inputs to
        # skip the cargo build entirely. `push` and `workflow_dispatch` always
        # run the check.
        name: UniFFI wrapper drift
        runs-on: macos-latest
        needs: wrapper-drift-changes
        if: |
            github.event_name != 'pull_request'
            || needs.wrapper-drift-changes.outputs.should-run == 'true'
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

            - uses: ./.github/actions/select-xcode

            - name: Read Rust toolchain version
              id: rust-toolchain
              run: |
                  version=$(awk -F'"' '/^channel/ {print $2; exit}' rust-toolchain.toml)
                  echo "version=$version" >> "$GITHUB_OUTPUT"

            - name: Install Rust toolchain
              uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
              with:
                  toolchain: ${{ steps.rust-toolchain.outputs.version }}
                  targets: aarch64-apple-darwin

            - name: Verify generated wrapper matches committed copy
              run: bash scripts/rust/check-wrapper-drift.sh

    docs:
        name: Documentation
        runs-on: macos-latest
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

            - uses: ./.github/actions/select-xcode

            - name: Verify documentation
              run: bash scripts/verify-docs.sh

    lint:
        name: Lint
        runs-on: macos-latest
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

            - uses: ./.github/actions/select-xcode

            - run: scripts/swift-format.sh

            - name: Suggest fixes (if check fails)
              if: failure()
              run: |
                  echo "### Here's how to fix the formatting locally:" >> $GITHUB_STEP_SUMMARY
                  echo "" >> $GITHUB_STEP_SUMMARY
                  echo '```bash' >> $GITHUB_STEP_SUMMARY
                  echo 'scripts/swift-format.sh --fix' >> $GITHUB_STEP_SUMMARY
                  echo '```' >> $GITHUB_STEP_SUMMARY

    rust-pin-guard:
        name: Rust pin guard
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

            - name: Check Rust source hash against rust/Pin.json
              run: bash scripts/rust/release/check-pin.sh

            - name: Check Package.swift inline pin matches rust/Pin.json
              run: bash scripts/rust/release/check-package-swift-pin.sh

            - name: Check for pre-release pin on main
              if: |
                  (github.event_name == 'pull_request' && github.base_ref == 'main')
                  || (github.event_name == 'push' && github.ref == 'refs/heads/main')
              env:
                  ALLOW_PRERELEASE_PIN: ${{ vars.ALLOW_PRERELEASE_PIN }}
              run: bash scripts/rust/release/check-prerelease-pin.sh