benchmarks.yml

name: Performance Benchmarks

on:
  workflow_dispatch:
    inputs:
      run_postgres:
        description: 'Run PostgreSQL benchmark (slow)'
        type: boolean
        default: true
  pull_request:
    branches: [main]
  push:
    branches: [main]

permissions:
  contents: read
  pull-requests: write

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Phase 0: Update PR comment to show benchmarks are running
  pr-comment-start:
    name: Create PR Comment
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    timeout-minutes: 2

    steps:
      - name: Find existing benchmark comment
        uses: peter-evans/find-comment@v3
        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '<!-- benchmark-results -->'

      - name: Get existing comment body
        if: steps.find-comment.outputs.comment-id != ''
        id: get-comment
        uses: actions/github-script@v7
        with:
          script: |
            const comment = await github.rest.issues.getComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.find-comment.outputs.comment-id }}
            });
            // Extract the results section (everything after the header and running message)
            const body = comment.data.body;
            // Remove any existing stale warning and running message
            let resultsSection = body
              .replace(/<!-- benchmark-results -->\n# Benchmark Comparison\n\n> ⚠️ \*\*Results below are stale\*\*[^\n]*\n\n/g, '')
              .replace(/<!-- benchmark-results -->\n# Benchmark Comparison\n\n/g, '')
              .replace(/⏳ \*\*Benchmarks are running\.\.\.\*\*\n\n---\n_Started at:[^_]*_\n\n---\n\n/g, '')
              .replace(/⏳ \*\*Benchmarks are running\.\.\.\*\*\n\n---\n_Started at:[^_]*_/g, '')
              .trim();

            // If there's actual content left (benchmark tables), save it
            if (resultsSection && resultsSection.includes('|')) {
              core.setOutput('has-results', 'true');
              core.setOutput('previous-results', resultsSection);
            } else {
              core.setOutput('has-results', 'false');
            }

      - name: Create new benchmark comment
        if: steps.find-comment.outputs.comment-id == ''
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: benchmark-results
          message: |
            <!-- benchmark-results -->
            # Benchmark Comparison

            ⏳ **Benchmarks are running...**

            This comment will be updated with the results when the benchmarks complete.

            ---
            _Started at: ${{ github.event.pull_request.updated_at }}_

      - name: Update existing benchmark comment with stale warning
        if: steps.find-comment.outputs.comment-id != '' && steps.get-comment.outputs.has-results == 'true'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: benchmark-results
          message: |
            <!-- benchmark-results -->
            # Benchmark Comparison

            > ⚠️ **Results below are stale** and not from the latest commit. This comment will be updated when CI completes on the latest run.

            ⏳ **Benchmarks are running...**

            ---
            _Started at: ${{ github.event.pull_request.updated_at }}_

            ---

            ${{ steps.get-comment.outputs.previous-results }}

      - name: Update existing benchmark comment without results
        if: steps.find-comment.outputs.comment-id != '' && steps.get-comment.outputs.has-results != 'true'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: benchmark-results
          message: |
            <!-- benchmark-results -->
            # Benchmark Comparison

            ⏳ **Benchmarks are running...**

            This comment will be updated with the results when the benchmarks complete.

            ---
            _Started at: ${{ github.event.pull_request.updated_at }}_

  # Phase 1: Build all packages
  build:
    name: Build Packages
    runs-on: ubuntu-latest
    timeout-minutes: 10

    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v2

      - uses: actions/setup-node@v4
        with:
          node-version: '22'
          cache: 'pnpm'

      - name: Install dependencies
        run: pnpm install

      - name: Build all packages
        run: pnpm build

      # Cache node_modules and package build outputs
      - name: Upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: build-artifacts
          path: |
            node_modules
            packages/*/dist
            workbench/node_modules
          retention-days: 1

  # Phase 2a: Starter (in-memory) benchmarks
  benchmark-starter:
    name: Benchmark Starter (in-memory)
    runs-on: ubuntu-latest
    needs: build
    timeout-minutes: 15

    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v2

      - uses: actions/setup-node@v4
        with:
          node-version: '22'

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
          path: .

      - name: Install dependencies
        run: pnpm install

      - name: Build workbench
        run: pnpm --filter @workflow-worlds/workbench build
        env:
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/starter"

      - name: Run benchmarks
        run: |
          cd workbench
          pnpm start &
          SERVER_PID=$!
          trap "kill $SERVER_PID 2>/dev/null" EXIT
          echo "Waiting for server to start..."
          sleep 10
          npx vitest bench --run --outputJson=../bench-results-starter.json
        env:
          DEPLOYMENT_URL: "http://localhost:3000"
          WORLD_NAME: starter
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/starter"

      - name: Render benchmark results
        uses: ./.github/actions/render-benchmarks
        with:
          benchmark-file: bench-results-starter.json
          world-name: starter

      - name: Upload benchmark results
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-starter
          path: |
            bench-results-starter.json
            workbench/bench-timings-starter.json

  # Phase 2b: MongoDB benchmarks
  benchmark-mongodb:
    name: Benchmark MongoDB
    runs-on: ubuntu-latest
    needs: build
    timeout-minutes: 15

    services:
      mongodb:
        image: mongo:7
        ports:
          - 27017:27017
        options: >-
          --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v2

      - uses: actions/setup-node@v4
        with:
          node-version: '22'

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
          path: .

      - name: Install dependencies
        run: pnpm install

      - name: Build workbench
        run: pnpm --filter @workflow-worlds/workbench build
        env:
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/mongodb"
          WORKFLOW_MONGODB_URI: "mongodb://localhost:27017/workflow"
          WORKFLOW_MONGODB_CHANGE_STREAMS: "false"

      - name: Run benchmarks
        run: |
          cd workbench
          pnpm start &
          SERVER_PID=$!
          trap "kill $SERVER_PID 2>/dev/null" EXIT
          echo "Waiting for server to start..."
          sleep 10
          npx vitest bench --run --outputJson=../bench-results-mongodb.json
        env:
          DEPLOYMENT_URL: "http://localhost:3000"
          WORLD_NAME: mongodb
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/mongodb"
          WORKFLOW_MONGODB_URI: "mongodb://localhost:27017/workflow"
          WORKFLOW_MONGODB_CHANGE_STREAMS: "false"

      - name: Render benchmark results
        uses: ./.github/actions/render-benchmarks
        with:
          benchmark-file: bench-results-mongodb.json
          world-name: mongodb

      - name: Upload benchmark results
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-mongodb
          path: |
            bench-results-mongodb.json
            workbench/bench-timings-mongodb.json

  # Phase 2c: Redis benchmarks
  benchmark-redis:
    name: Benchmark Redis
    runs-on: ubuntu-latest
    needs: build
    timeout-minutes: 15

    services:
      redis:
        image: redis:7-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v2

      - uses: actions/setup-node@v4
        with:
          node-version: '22'

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
          path: .

      - name: Install dependencies
        run: pnpm install

      - name: Build workbench
        run: pnpm --filter @workflow-worlds/workbench build
        env:
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/redis"
          WORKFLOW_REDIS_URI: "redis://localhost:6379"

      - name: Run benchmarks
        run: |
          echo "Flushing Redis..."
          echo "FLUSHALL" | nc -q 1 localhost 6379 || true
          sleep 1

          cd workbench
          echo "Starting server..."
          pnpm start &
          SERVER_PID=$!
          trap "kill $SERVER_PID 2>/dev/null" EXIT

          echo "Waiting for server to start..."
          for i in {1..30}; do
            if curl -s http://localhost:3000/health > /dev/null 2>&1; then
              echo "Server is ready after ${i}s"
              break
            fi
            echo "Waiting... ($i/30)"
            sleep 1
          done

          echo "Running vitest benchmarks..."
          npx vitest bench --run --outputJson=../bench-results-redis.json
        env:
          DEPLOYMENT_URL: "http://localhost:3000"
          WORLD_NAME: redis
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/redis"
          WORKFLOW_REDIS_URI: "redis://localhost:6379"

      - name: Render benchmark results
        uses: ./.github/actions/render-benchmarks
        with:
          benchmark-file: bench-results-redis.json
          world-name: redis

      - name: Upload benchmark results
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-redis
          path: |
            bench-results-redis.json
            workbench/bench-timings-redis.json

  # Phase 2d: Turso (libSQL/SQLite) benchmarks
  benchmark-turso:
    name: Benchmark Turso
    runs-on: ubuntu-latest
    needs: build
    timeout-minutes: 15

    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v2

      - uses: actions/setup-node@v4
        with:
          node-version: '22'

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
          path: .

      - name: Install dependencies
        run: pnpm install

      - name: Setup Turso database
        run: pnpm --filter @workflow-worlds/workbench exec workflow-turso-setup
        env:
          WORKFLOW_TURSO_DATABASE_URL: "file:benchmark.db"

      - name: Build workbench
        run: pnpm --filter @workflow-worlds/workbench build
        env:
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/turso"
          WORKFLOW_TURSO_DATABASE_URL: "file:benchmark.db"

      - name: Run benchmarks
        run: |
          cd workbench
          pnpm start &
          SERVER_PID=$!
          trap "kill $SERVER_PID 2>/dev/null" EXIT
          echo "Waiting for server to start..."
          sleep 10
          npx vitest bench --run --outputJson=../bench-results-turso.json
        env:
          DEPLOYMENT_URL: "http://localhost:3000"
          WORLD_NAME: turso
          WORKFLOW_TARGET_WORLD: "@workflow-worlds/turso"
          WORKFLOW_TURSO_DATABASE_URL: "file:benchmark.db"

      - name: Render benchmark results
        uses: ./.github/actions/render-benchmarks
        with:
          benchmark-file: bench-results-turso.json
          world-name: turso

      - name: Upload benchmark results
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-turso
          path: |
            bench-results-turso.json
            workbench/bench-timings-turso.json

  # Phase 2e: PostgreSQL benchmarks (skipped on PRs by default - slow)
  benchmark-postgres:
    name: Benchmark PostgreSQL
    runs-on: ubuntu-latest
    needs: build
    # Run on push to main, or manual trigger with run_postgres=true (skips PRs by default)
    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.run_postgres)
    timeout-minutes: 20

    services:
      postgres:
        image: postgres:16-alpine
        ports:
          - 5432:5432
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: workflow
        options: >-
          --health-cmd "pg_isready -U postgres"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v2

      - uses: actions/setup-node@v4
        with:
          node-version: '22'

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
          path: .

      - name: Install dependencies
        run: pnpm install

      - name: Setup PostgreSQL schema
        run: pnpm --filter @workflow-worlds/workbench exec workflow-postgres-setup
        env:
          WORKFLOW_POSTGRES_URL: "postgres://postgres:postgres@localhost:5432/workflow"

      - name: Build workbench
        run: pnpm --filter @workflow-worlds/workbench build
        env:
          WORKFLOW_TARGET_WORLD: "@workflow/world-postgres"
          WORKFLOW_POSTGRES_URL: "postgres://postgres:postgres@localhost:5432/workflow"

      - name: Run benchmarks
        run: |
          cd workbench
          pnpm start &
          SERVER_PID=$!
          trap "kill $SERVER_PID 2>/dev/null" EXIT
          echo "Waiting for server to start..."
          sleep 10
          npx vitest bench --run --outputJson=../bench-results-postgres.json
        env:
          DEPLOYMENT_URL: "http://localhost:3000"
          WORLD_NAME: postgres
          WORKFLOW_TARGET_WORLD: "@workflow/world-postgres"
          WORKFLOW_POSTGRES_URL: "postgres://postgres:postgres@localhost:5432/workflow"

      - name: Render benchmark results
        uses: ./.github/actions/render-benchmarks
        with:
          benchmark-file: bench-results-postgres.json
          world-name: postgres

      - name: Upload benchmark results
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-postgres
          path: |
            bench-results-postgres.json
            workbench/bench-timings-postgres.json

  # Phase 2f: Local (built-in) world benchmarks
  benchmark-local:
    name: Benchmark Local (built-in)
    runs-on: ubuntu-latest
    needs: build
    timeout-minutes: 15

    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v2

      - uses: actions/setup-node@v4
        with:
          node-version: '22'

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
          path: .

      - name: Install dependencies
        run: pnpm install

      - name: Build workbench
        run: pnpm --filter @workflow-worlds/workbench build

      - name: Run benchmarks
        run: |
          cd workbench
          pnpm start &
          SERVER_PID=$!
          trap "kill $SERVER_PID 2>/dev/null" EXIT
          echo "Waiting for server to start..."
          sleep 10
          npx vitest bench --run --outputJson=../bench-results-local.json
        env:
          DEPLOYMENT_URL: "http://localhost:3000"
          WORLD_NAME: local

      - name: Render benchmark results
        uses: ./.github/actions/render-benchmarks
        with:
          benchmark-file: bench-results-local.json
          world-name: local

      - name: Upload benchmark results
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-local
          path: |
            bench-results-local.json
            workbench/bench-timings-local.json

  # Phase 3: Aggregate all benchmark results and create comparison
  summary:
    name: Benchmark Summary
    runs-on: ubuntu-latest
    needs: [benchmark-starter, benchmark-mongodb, benchmark-redis, benchmark-turso, benchmark-postgres, benchmark-local]
    if: always() && !cancelled()
    timeout-minutes: 10

    steps:
      - uses: actions/checkout@v4

      - name: Download all benchmark artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: bench-results-*
          path: benchmark-results
          merge-multiple: true

      - name: List downloaded files
        run: find benchmark-results -type f -name "*.json" | sort

      # For PRs, try to download baseline results from the latest main branch run
      - name: Download baseline from main branch
        if: github.event_name == 'pull_request'
        continue-on-error: true
        uses: dawidd6/action-download-artifact@v6
        with:
          workflow: benchmarks.yml
          branch: main
          name: baseline-benchmark-results
          path: baseline-results
          search_artifacts: true
          if_no_artifact_found: warn

      - name: Aggregate and compare benchmarks
        id: aggregate
        run: |
          # Check if baseline results exist
          BASELINE_ARG=""
          if [ -d "baseline-results" ] && [ "$(ls -A baseline-results 2>/dev/null)" ]; then
            echo "Found baseline results from main branch"
            BASELINE_ARG="--baseline baseline-results"
          else
            echo "No baseline results found, showing results without comparison"
          fi
          # Capture output to both file and step summary
          node .github/scripts/aggregate-benchmarks.js benchmark-results $BASELINE_ARG | tee benchmark-summary.md >> $GITHUB_STEP_SUMMARY

      - name: Check benchmark job statuses
        id: check-status
        run: |
          # Check if any benchmark jobs failed
          STARTER_STATUS="${{ needs.benchmark-starter.result }}"
          MONGODB_STATUS="${{ needs.benchmark-mongodb.result }}"
          REDIS_STATUS="${{ needs.benchmark-redis.result }}"
          TURSO_STATUS="${{ needs.benchmark-turso.result }}"
          POSTGRES_STATUS="${{ needs.benchmark-postgres.result }}"
          LOCAL_STATUS="${{ needs.benchmark-local.result }}"

          echo "starter=$STARTER_STATUS" >> $GITHUB_OUTPUT
          echo "mongodb=$MONGODB_STATUS" >> $GITHUB_OUTPUT
          echo "redis=$REDIS_STATUS" >> $GITHUB_OUTPUT
          echo "turso=$TURSO_STATUS" >> $GITHUB_OUTPUT
          echo "postgres=$POSTGRES_STATUS" >> $GITHUB_OUTPUT
          echo "local=$LOCAL_STATUS" >> $GITHUB_OUTPUT

          if [[ "$STARTER_STATUS" == "failure" || "$MONGODB_STATUS" == "failure" || "$REDIS_STATUS" == "failure" || "$TURSO_STATUS" == "failure" || "$POSTGRES_STATUS" == "failure" || "$LOCAL_STATUS" == "failure" ]]; then
            echo "has_failures=true" >> $GITHUB_OUTPUT
          else
            echo "has_failures=false" >> $GITHUB_OUTPUT
          fi

      - name: Update PR comment with results
        if: github.event_name == 'pull_request'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: benchmark-results
          path: benchmark-summary.md

      - name: Append failure notice to PR comment
        if: github.event_name == 'pull_request' && steps.check-status.outputs.has_failures == 'true'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: benchmark-results
          append: true
          message: |

            ---
            Some benchmark jobs failed:
            - Local: ${{ needs.benchmark-local.result }}
            - Starter: ${{ needs.benchmark-starter.result }}
            - MongoDB: ${{ needs.benchmark-mongodb.result }}
            - Redis: ${{ needs.benchmark-redis.result }}
            - Turso: ${{ needs.benchmark-turso.result }}
            - PostgreSQL: ${{ needs.benchmark-postgres.result }}

            Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.

      # On main branch, save results as baseline for future PR comparisons
      - name: Upload baseline results
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        uses: actions/upload-artifact@v4
        with:
          name: baseline-benchmark-results
          path: benchmark-results/
          retention-days: 90