main-workflow.yml

name: Main Workflow - Lint, Build, Test

on:
  push:
    branches:
      - master
      - main
    tags:
      - "**"
  pull_request:
  workflow_dispatch:
    inputs:
      python-version:
        description: "Python version to use"
        required: false
        default: "3.12" # PY_VER
      rust-version:
        description: "Rust version to use"
        required: false
        default: "1.91" # RUST_VER

env:
  PYTHON_VERSION: ${{ inputs.python-version || '3.12' }} # PY_VER
  RUST_VERSION: ${{ inputs.rust-version || '1.91' }} # RUST_VER

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

permissions: {} # workflow-level default — deny all

jobs:
  # Setup Python ======
  python-env:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Cache pip and Poetry virtualenv
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
          lookup-only: true

      - name: Install Poetry
        run: pip3 install poetry

      - name: Install Python dependencies
        run: poetry install --with tokenserver-unit-tests,dev --no-interaction --no-ansi

      - name: Display Python Version Info
        run: |
          if [ "$(which python)" != "" ]; then python --version; fi
          uname -a
          cat /etc/os-release
  # Setup Rust ======
  rust-env:
    runs-on: ubuntu-latest
    permissions: {}
    steps:
      - name: Cache Rust toolchain
        id: cache-rust-toolchain
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
          key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}
          lookup-only: true

      - name: Install Rust toolchain
        if: steps.cache-rust-toolchain.outputs.cache-hit != 'true'
        run: rustup toolchain install ${RUST_VERSION} --component rustfmt --component clippy --component llvm-tools-preview --no-self-update && rustup default ${RUST_VERSION}

      - name: Display Rust Version Info
        shell: bash
        run: |
          if [ "$(which rustc)" != "" ]; then rustc --version; fi
          uname -a
          cat /etc/os-release

  # Python lint and format checks ======
  python-checks:
    needs: python-env
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Restore pip and Poetry virtualenv
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}

      - name: Install Poetry
        run: pip3 install poetry

      - name: Python Format Check
        run: poetry run ruff format --diff tools

      - name: Python Lint Check
        run: poetry run ruff check tools

      - name: Python Docstring Check
        run: poetry run pydocstyle -es --count --config=pyproject.toml tools

      - name: Python Security Check
        run: poetry run bandit --quiet -r -c pyproject.toml tools

      - name: Python Type Check
        run: poetry run mypy --config-file=pyproject.toml tools
  # Rust lint and format checks ======
  rust-checks:
    needs: rust-env
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Restore Rust toolchain
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
          key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}

      - name: Set Rust toolchain
        run: rustup default ${RUST_VERSION}

      - name: Cache cargo-audit
        id: cache-cargo-audit
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: ~/.cargo/bin/cargo-audit
          key: ${{ runner.os }}-cargo-audit-${{ hashFiles('.github/workflows/main-workflow.yml') }}

      - name: Install cargo-audit
        if: steps.cache-cargo-audit.outputs.cache-hit != 'true'
        run: cargo install --locked cargo-audit

      - name: Cache mdbook
        id: cache-mdbook
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cargo/bin/mdbook
            ~/.cargo/bin/mdbook-mermaid
          key: ${{ runner.os }}-mdbook-${{ hashFiles('Makefile') }}

      - name: Setup documentation checks
        if: steps.cache-mdbook.outputs.cache-hit != 'true'
        run: make doc-install-deps

      - name: Rust Format Check
        run: cargo fmt -- --check

      - name: Cargo Audit
        run: cargo audit

      - name: Documentation checks
        run: make doc-test
  # Rust clippy lint checker
  clippy:
    needs: rust-env
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      matrix:
        target: [spanner, mysql, postgres]
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Restore Rust toolchain
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
          key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}

      - name: Set Rust toolchain
        run: rustup default ${RUST_VERSION}

      - name: Rust Clippy ${{ matrix.target }}
        run: make clippy_${{ matrix.target }}

  # Rust clippy release mode lint checker
  clippy-release:
    needs: rust-env
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      matrix:
        target: [spanner, mysql, postgres]
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Restore Rust toolchain
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
          key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}

      - name: Restore Rust cache
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-release-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-release-${{ matrix.target }}-

      - name: Set Rust toolchain
        run: rustup default ${RUST_VERSION}

      - name: Rust Clippy release ${{ matrix.target }}
        run: make clippy_release_${{ matrix.target }}

  # Postgres unit tests ======
  build-and-unit-test-postgres:
    needs: [rust-env, python-env]
    runs-on: ubuntu-latest
    permissions:
      contents: read
      checks: write

    services:
      postgres:
        image: postgres:18.0
        env:
          POSTGRES_USER: test
          POSTGRES_PASSWORD: test
          POSTGRES_DB: syncstorage
        ports:
          - 5432:5432
        options: >-
          --health-cmd="pg_isready -U test"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5

    env:
      SYNC_SYNCSTORAGE__DATABASE_URL: postgres://test:test@127.0.0.1/syncstorage
      SYNC_TOKENSERVER__DATABASE_URL: postgres://test:test@127.0.0.1/tokenserver
      SYNC_TOKENSERVER__NODE_TYPE: postgres
      RUST_BACKTRACE: 1
      RUST_TEST_THREADS: 1
      CARGO_INCREMENTAL: "0"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Restore Rust toolchain
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
          key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}

      - name: Set Rust toolchain
        run: rustup default ${RUST_VERSION}

      - name: Restore pip and Poetry virtualenv
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}

      - name: Cache Cargo build artifacts
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-postgres-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-postgres-

      - name: Create test results directory
        run: mkdir -p workflow/test-results

      - name: Install PostgreSQL client
        run: sudo apt-get update && sudo apt-get install -y postgresql-client

      - name: Create Tokenserver database
        run: |
          PGPASSWORD=test psql -U test -h 127.0.0.1 -d syncstorage -c 'CREATE DATABASE tokenserver;'

      - name: Create version.json
        run: |
          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
            "${GITHUB_SHA}" \
            "${GITHUB_REF_NAME}" \
            "${GITHUB_REPOSITORY_OWNER}" \
            "${GITHUB_REPOSITORY_NAME}" \
            "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
            > version.json
        env:
          GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}

      - name: Cache cargo-nextest
        id: cache-cargo-nextest-postgres
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: ~/.cargo/bin/cargo-nextest
          key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }}

      - name: Install cargo-nextest
        if: steps.cache-cargo-nextest-postgres.outputs.cache-hit != 'true'
        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin

      - name: Cache cargo-llvm-cov
        id: cache-cargo-llvm-cov-postgres
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: ~/.cargo/bin/cargo-llvm-cov
          key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }}

      - name: Install cargo-llvm-cov
        if: steps.cache-cargo-llvm-cov-postgres.outputs.cache-hit != 'true'
        run: cargo install --locked cargo-llvm-cov

      - name: Run unit tests with coverage
        run: make postgres_test_with_coverage

      - name: Run unit tests with coverage (quota enforced)
        run: make postgres_test_with_coverage
        env:
          SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1

      - name: Install Poetry
        run: pip3 install poetry

      - name: Run Postgres utils tests
        working-directory: tools/postgres
        run: |
          poetry install --no-interaction --no-ansi
          WORKFLOW=$(echo "${GITHUB_WORKFLOW}" | tr ' ' '-' | tr '[:upper:]' '[:lower:]')
          poetry run pytest test_purge_ttl.py -v --junit-xml="../../workflow/test-results/${GITHUB_RUN_NUMBER}__$(date +%s)__$(basename ${GITHUB_REPOSITORY})__${WORKFLOW}__postgres_utils__results.xml"
        env:
          SYNC_SYNCSTORAGE__DATABASE_URL: postgresql://test:test@127.0.0.1/syncstorage

      - name: Publish Test Report
        uses: dorny/test-reporter@29672c52a8220c09bd9f79b5b6cf6315f1763996
        if: always()
        with:
          name: Postgres Unit Tests
          path: workflow/test-results/*.xml
          reporter: java-junit
          fail-on-error: false

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: postgres-test-results
          path: workflow/test-results/

      # Upload to GCS on master
      - name: Authenticate to Google Cloud
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}

      - name: Upload JUnit results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
          glob: "*.xml"
          parent: false
          process_gcloudignore: false

      - name: Upload coverage results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage
          glob: "*.json"
          parent: false
          process_gcloudignore: false
  # Docker build Postgres Image ======
  build-postgres-image:
    runs-on: ubuntu-latest
    needs: [rust-env, python-env]
    permissions:
      contents: read
      actions: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Create version.json
        run: |
          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
            "${GITHUB_SHA}" \
            "${GITHUB_REF_NAME}" \
            "${GITHUB_REPOSITORY_OWNER}" \
            "${GITHUB_REPOSITORY_NAME}" \
            "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
            > version.json
        env:
          GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}

      - name: Cache Docker image tar
        id: cache-postgres-image
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: /tmp/postgres-image.tar
          key: ${{ runner.os }}-postgres-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }}

      - name: Set up Docker Buildx
        if: steps.cache-postgres-image.outputs.cache-hit != 'true'
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4

      - name: Build Postgres Docker image
        if: steps.cache-postgres-image.outputs.cache-hit != 'true'
        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7
        with:
          context: .
          push: false
          tags: app:build
          build-args: |
            SYNCSTORAGE_DATABASE_BACKEND=postgres
            TOKENSERVER_DATABASE_BACKEND=postgres
          outputs: type=docker,dest=/tmp/postgres-image.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Upload Docker image artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: postgres-docker-image
          path: /tmp/postgres-image.tar
          retention-days: 1
  # End-to-end tests for Postgres build ======
  postgres-e2e-tests:
    runs-on: ubuntu-latest
    needs: build-postgres-image
    permissions:
      contents: read
      checks: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: postgres-docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/postgres-image.tar

      - name: Create test results directory
        run: mkdir -p workflow/test-results

      - name: Run Postgres e2e tests
        run: make docker_run_postgres_e2e_tests
        env:
          SYNCSTORAGE_RS_IMAGE: app:build

      - name: Publish E2E Test Report
        uses: dorny/test-reporter@29672c52a8220c09bd9f79b5b6cf6315f1763996
        if: always()
        with:
          name: Postgres E2E Tests
          path: workflow/test-results/*.xml
          reporter: java-junit
          fail-on-error: false

      - name: Upload e2e test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: postgres-e2e-test-results
          path: workflow/test-results/

      # Upload to GCS on master
      - name: Authenticate to Google Cloud
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}

      - name: Upload e2e test results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
          glob: "*.xml"
          parent: false
          process_gcloudignore: false

  # MySQL Unit tests ======
  build-and-unit-test-mysql:
    runs-on: ubuntu-latest
    needs: [rust-env, python-env]
    permissions:
      contents: read
      checks: write

    services:
      mysql:
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: password
          MYSQL_USER: test
          MYSQL_PASSWORD: test
          MYSQL_DATABASE: syncstorage
        ports:
          - 3306:3306
        options: >-
          --health-cmd="mysqladmin ping"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5

    env:
      SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@127.0.0.1/syncstorage
      SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver
      SYNC_TOKENSERVER__NODE_TYPE: spanner
      RUST_BACKTRACE: 1
      RUST_TEST_THREADS: 1
      CARGO_INCREMENTAL: "0"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Restore Rust toolchain
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
          key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}

      - name: Set Rust toolchain
        run: rustup default ${RUST_VERSION}

      - name: Restore pip and Poetry virtualenv
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}

      - name: Cache Cargo build artifacts
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-mysql-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-mysql-

      - name: Create test results directory
        run: mkdir -p workflow/test-results

      - name: Install MySQL client
        run: sudo apt-get update && sudo apt-get install -y default-mysql-client

      - name: Create Tokenserver database
        run: |
          mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;'
          mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';"

      - name: Create version.json
        run: |
          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
            "${GITHUB_SHA}" \
            "${GITHUB_REF_NAME}" \
            "${GITHUB_REPOSITORY_OWNER}" \
            "${GITHUB_REPOSITORY_NAME}" \
            "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
            > version.json
        env:
          GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}

      - name: Cache cargo-nextest
        id: cache-cargo-nextest-mysql
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: ~/.cargo/bin/cargo-nextest
          key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }}

      - name: Install cargo-nextest
        if: steps.cache-cargo-nextest-mysql.outputs.cache-hit != 'true'
        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin

      - name: Cache cargo-llvm-cov
        id: cache-cargo-llvm-cov-mysql
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: ~/.cargo/bin/cargo-llvm-cov
          key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }}

      - name: Install cargo-llvm-cov
        if: steps.cache-cargo-llvm-cov-mysql.outputs.cache-hit != 'true'
        run: cargo install --locked cargo-llvm-cov

      - name: Run unit tests with coverage
        run: make test_with_coverage

      - name: Run unit tests with coverage (quota enforced)
        run: make test_with_coverage
        env:
          SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1

      - name: Publish Test Report
        uses: dorny/test-reporter@29672c52a8220c09bd9f79b5b6cf6315f1763996
        if: always()
        with:
          name: MySQL Unit Tests
          path: workflow/test-results/*.xml
          reporter: java-junit
          fail-on-error: false

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: mysql-test-results
          path: workflow/test-results/

      # Upload to GCS on master
      - name: Authenticate to Google Cloud
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}

      - name: Upload JUnit results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
          glob: "*.xml"
          parent: false
          process_gcloudignore: false

      - name: Upload coverage results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage
          glob: "*.json"
          parent: false
          process_gcloudignore: false

  # Docker build for MySQL Image
  build-mysql-image:
    runs-on: ubuntu-latest
    needs: [rust-env, python-env]
    permissions:
      contents: read
      actions: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Create version.json
        run: |
          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
            "${GITHUB_SHA}" \
            "${GITHUB_REF_NAME}" \
            "${GITHUB_REPOSITORY_OWNER}" \
            "${GITHUB_REPOSITORY_NAME}" \
            "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
            > version.json
        env:
          GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}

      - name: Cache Docker image tar
        id: cache-mysql-image
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: /tmp/mysql-image.tar
          key: ${{ runner.os }}-mysql-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }}

      - name: Set up Docker Buildx
        if: steps.cache-mysql-image.outputs.cache-hit != 'true'
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4

      - name: Build MySQL Docker image
        if: steps.cache-mysql-image.outputs.cache-hit != 'true'
        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7
        with:
          context: .
          push: false
          tags: app:build
          build-args: |
            SYNCSTORAGE_DATABASE_BACKEND=mysql
            TOKENSERVER_DATABASE_BACKEND=mysql
          outputs: type=docker,dest=/tmp/mysql-image.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Upload Docker image artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: mysql-docker-image
          path: /tmp/mysql-image.tar
          retention-days: 1
  # End-to-end tests for MySQL ======
  mysql-e2e-tests:
    runs-on: ubuntu-latest
    needs: build-mysql-image
    permissions:
      contents: read
      checks: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: mysql-docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/mysql-image.tar

      - name: Create test results directory
        run: mkdir -p workflow/test-results

      - name: Run MySQL e2e tests
        run: make docker_run_mysql_e2e_tests
        env:
          SYNCSTORAGE_RS_IMAGE: app:build

      - name: Publish E2E Test Report
        uses: dorny/test-reporter@29672c52a8220c09bd9f79b5b6cf6315f1763996
        if: always()
        with:
          name: MySQL E2E Tests
          path: workflow/test-results/*.xml
          reporter: java-junit
          fail-on-error: false

      - name: Upload e2e test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: mysql-e2e-test-results
          path: workflow/test-results/

      # Upload to GCS on master
      - name: Authenticate to Google Cloud
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}

      - name: Upload e2e test results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
          glob: "*.xml"
          parent: false
          process_gcloudignore: false

  # Spanner Unit tests ======
  build-and-unit-test-spanner:
    runs-on: ubuntu-latest
    needs: [rust-env, python-env]
    permissions:
      contents: read
      checks: write

    services:
      spanner-emulator:
        # SPANNER_EMULATOR_VER (unit tests) — keep in sync with docker/docker-compose.spanner.yaml
        image: gcr.io/cloud-spanner-emulator/emulator:1.5.52
        ports:
          - 9010:9010
          - 9020:9020

      mysql:
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: password
          MYSQL_USER: test
          MYSQL_PASSWORD: test
          MYSQL_DATABASE: syncstorage
        ports:
          - 3306:3306
        options: >-
          --health-cmd="mysqladmin ping"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=3

    env:
      # The code expects a spanner URL like:
      SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
      RUST_BACKTRACE: 1
      RUST_TEST_THREADS: 1
      CARGO_INCREMENTAL: "0"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Restore Rust toolchain
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
          key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}

      - name: Set Rust toolchain
        run: rustup default ${RUST_VERSION}

      - name: Restore pip and Poetry virtualenv
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}

      - name: Cache Cargo build artifacts
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-spanner-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-spanner-

      - name: Create test results directory
        run: mkdir -p workflow/test-results

      - name: Create version.json
        run: |
          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
            "${GITHUB_SHA}" \
            "${GITHUB_REF_NAME}" \
            "${GITHUB_REPOSITORY_OWNER}" \
            "${GITHUB_REPOSITORY_NAME}" \
            "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
            > version.json
        env:
          GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}

      - name: Cache cargo-nextest
        id: cache-cargo-nextest-spanner
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: ~/.cargo/bin/cargo-nextest
          key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }}

      - name: Install cargo-nextest
        if: steps.cache-cargo-nextest-spanner.outputs.cache-hit != 'true'
        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin

      - name: Cache cargo-llvm-cov
        id: cache-cargo-llvm-cov-spanner
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: ~/.cargo/bin/cargo-llvm-cov
          key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }}

      - name: Install cargo-llvm-cov
        if: steps.cache-cargo-llvm-cov-spanner.outputs.cache-hit != 'true'
        run: cargo install --locked cargo-llvm-cov

      - name: Build workspace (spanner feature)
        run: |
          # Build with the spanner feature so any compile-time issues surface early
          cargo build --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier

      - name: Wait for Spanner Emulator to be ready
        run: |
          echo "Waiting for Spanner emulator to be ready..."
          for i in {1..30}; do
            if curl -s http://localhost:9020/ > /dev/null 2>&1; then
              echo "Spanner emulator is ready (REST port 9020 responding)"
              break
            fi
            echo "Attempt $i/30: Spanner emulator not ready yet, waiting..."
            sleep 2
          done
          # Verify both ports are accessible
          if ! curl -s http://localhost:9020/ > /dev/null 2>&1; then
            echo "ERROR: Cannot connect to Spanner emulator REST API at localhost:9020"
            exit 1
          fi
          echo "Spanner emulator is fully ready"

      - name: Setup Spanner schema & instance (prepare-spanner.sh)
        env:
          SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
          SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: http://localhost:9020
        run: |
          # prepare-spanner.sh uses the REST API (port 9020)
          scripts/prepare-spanner.sh

      - name: Create Tokenserver database
        run: |
          mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;'
          mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';"

      - name: Run Spanner unit tests with coverage
        env:
          SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
          SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: localhost:9010
          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver
          SYNC_TOKENSERVER__NODE_TYPE: spanner
          RUST_TEST_THREADS: 1
        run: make spanner_test_with_coverage

      - name: Publish Test Report
        uses: dorny/test-reporter@29672c52a8220c09bd9f79b5b6cf6315f1763996
        if: always()
        with:
          name: Spanner Unit Tests
          path: workflow/test-results/*.xml
          reporter: java-junit
          fail-on-error: false

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: spanner-test-results
          path: workflow/test-results/

      # Upload to GCS on master
      - name: Authenticate to Google Cloud
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}

      - name: Upload JUnit results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
          glob: "*.xml"
          parent: false
          process_gcloudignore: false

      - name: Upload coverage results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage
          glob: "*.json"
          parent: false
          process_gcloudignore: false

  # Docker image build for Spanner
  build-spanner-image:
    runs-on: ubuntu-latest
    needs: [rust-env, python-env]
    permissions:
      contents: read
      actions: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Create version.json
        run: |
          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
            "${GITHUB_SHA}" \
            "${GITHUB_REF_NAME}" \
            "${GITHUB_REPOSITORY_OWNER}" \
            "${GITHUB_REPOSITORY_NAME}" \
            "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
            > version.json
        env:
          GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}

      - name: Cache Docker image tar
        id: cache-spanner-image
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
        with:
          path: /tmp/spanner-image.tar
          key: ${{ runner.os }}-spanner-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }}

      - name: Set up Docker Buildx
        if: steps.cache-spanner-image.outputs.cache-hit != 'true'
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4

      - name: Build Spanner Docker image (local artifact)
        if: steps.cache-spanner-image.outputs.cache-hit != 'true'
        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7
        with:
          context: .
          push: false
          tags: app:build
          build-args: |
            SYNCSTORAGE_DATABASE_BACKEND=spanner
          outputs: type=docker,dest=/tmp/spanner-image.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Upload Docker image artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: spanner-docker-image
          path: /tmp/spanner-image.tar
          retention-days: 1

  # End-to-end tests for Spanner ======
  spanner-e2e-tests:
    runs-on: ubuntu-latest
    needs: build-spanner-image
    permissions:
      contents: read
      checks: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: spanner-docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/spanner-image.tar

      - name: Create test results directory
        run: mkdir -p workflow/test-results

      - name: Run Spanner e2e tests
        run: make docker_run_spanner_e2e_tests
        env:
          SYNCSTORAGE_RS_IMAGE: app:build

      - name: Publish E2E Test Report
        uses: dorny/test-reporter@29672c52a8220c09bd9f79b5b6cf6315f1763996
        if: always()
        with:
          name: Spanner E2E Tests
          path: workflow/test-results/*.xml
          reporter: java-junit
          fail-on-error: false

      - name: Upload e2e test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: spanner-e2e-test-results
          path: workflow/test-results/

      # Upload to GCS on master
      - name: Authenticate to Google Cloud
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}

      - name: Upload e2e test results to GCS
        if: github.ref == 'refs/heads/master'
        uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # v3.0.0
        with:
          path: workflow/test-results
          destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
          glob: "*.xml"
          parent: false
          process_gcloudignore: false