test.yml

name: test

on:
  pull_request:
    branches:
      - main
      - next/*
  # Also run on pushes to main so the pnpm store cache is populated on the
  # default branch. GitHub only lets branches inherit caches created on the
  # default branch, so without this every PR branch starts with a cold cache
  # and re-downloads all dependencies from scratch.
  push:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # ---------------------------------------------------------------------------
  # Detects whether anything outside the ignore list below changed. Meta files
  # (agent instructions, docs, license, .gitignore) cannot affect lint,
  # packaging or test outcomes, so a change touching only those skips the whole
  # matrix.
  #
  # The filter runs inside a job rather than as an `on: paths-ignore` trigger:
  # a workflow skipped by a trigger-level path filter never reports a status, so
  # required checks stay pending forever and block merge. The CI gates below
  # depend on this job, so if it fails, the gates fail rather than passing on a
  # matrix that never ran.
  # ---------------------------------------------------------------------------
  changes:
    name: detect changes
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      code: ${{ steps.filter.outputs.code }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          # 'every' makes a file match `code` only when it matches *all* the
          # patterns, i.e. when it is none of the ignored paths. `code` is
          # therefore true when at least one changed file lives outside the
          # ignore list.
          predicate-quantifier: 'every'
          filters: |
            code:
              - '!.claude/**'
              - '!AGENTS.md'
              - '!CLAUDE.md'
              - '!CONTRIBUTING.md'
              - '!LICENSE'
              - '!README.md'
              - '!.gitignore'

  lint:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      # Test that the packaging works as well
      - run: pnpm -r pack
      - run: pnpm run lint

  # ---------------------------------------------------------------------------
  # Unit tests
  # ---------------------------------------------------------------------------
  test-ubuntu:
    name: test - ubuntu
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: pnpm run prepack
      - run: pnpm run test
      - name: Save LLM rules as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: llm-rules-test-ubuntu
          if-no-files-found: error
          path: packages/cli/dist/ai-context/*
          retention-days: 1
  test-windows:
    name: test - windows
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: blacksmith-4vcpu-windows-2025
    steps:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: pnpm run prepack
      - run: pnpm run test
      - name: Save LLM rules as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: llm-rules-test-windows
          if-no-files-found: error
          path: packages/cli/dist/ai-context/*
          retention-days: 1

  # ---------------------------------------------------------------------------
  # checkly e2e — hits the live Checkly API (PRs only). Windows is sharded
  # because it is the slow leg; Ubuntu runs the whole suite in one job.
  # ---------------------------------------------------------------------------
  test-e2e-checkly-ubuntu:
    name: e2e - checkly - ubuntu
    needs: changes
    if: github.event_name == 'pull_request' && needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: pnpm run prepack
      - run: pnpm --filter checkly run test:e2e
        env:
          CHECKLY_ACCOUNT_NAME: ${{ secrets.E2E_CHECKLY_ACCOUNT_NAME }}
          CHECKLY_ACCOUNT_ID: ${{ secrets.E2E_CHECKLY_ACCOUNT_ID }}
          CHECKLY_API_KEY: ${{ secrets.E2E_CHECKLY_API_KEY }}
          CHECKLY_EMPTY_ACCOUNT_ID: ${{ secrets.E2E_EMPTY_CHECKLY_ACCOUNT_ID }}
          CHECKLY_EMPTY_API_KEY: ${{ secrets.E2E_EMPTY_CHECKLY_API_KEY }}
  test-e2e-checkly-windows:
    name: e2e - checkly - windows (${{ matrix.shard }})
    needs: changes
    if: github.event_name == 'pull_request' && needs.changes.outputs.code == 'true'
    strategy:
      fail-fast: false
      matrix:
        shard: ['1/4', '2/4', '3/4', '4/4']
    runs-on: blacksmith-4vcpu-windows-2025
    steps:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: pnpm run prepack
      # Deliberately no `--` separator: pnpm appends trailing args to the
      # script command as-is, so --shard reaches vitest without one. Unlike
      # npm, pnpm forwards a literal `--` verbatim into the script, where
      # vitest's parser collects everything after it into an argument list it
      # never reads — silently discarding --shard. The suite still passes when
      # that happens, so a green run is not evidence the flag landed; check
      # that each shard reports fewer test files than the full suite and that
      # the four counts sum to it.
      - run: pnpm --filter checkly run test:e2e --shard=${{ matrix.shard }}
        env:
          CHECKLY_ACCOUNT_NAME: ${{ secrets.E2E_CHECKLY_ACCOUNT_NAME }}
          CHECKLY_ACCOUNT_ID: ${{ secrets.E2E_CHECKLY_ACCOUNT_ID }}
          CHECKLY_API_KEY: ${{ secrets.E2E_CHECKLY_API_KEY }}
          CHECKLY_EMPTY_ACCOUNT_ID: ${{ secrets.E2E_EMPTY_CHECKLY_ACCOUNT_ID }}
          CHECKLY_EMPTY_API_KEY: ${{ secrets.E2E_EMPTY_CHECKLY_API_KEY }}

  # ---------------------------------------------------------------------------
  # create-checkly e2e (PRs only)
  # ---------------------------------------------------------------------------
  test-e2e-create-checkly-ubuntu:
    name: e2e - create-checkly - ubuntu
    needs: changes
    if: github.event_name == 'pull_request' && needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: pnpm --filter create-checkly run test:e2e
  test-e2e-create-checkly-windows:
    name: e2e - create-checkly - windows
    needs: changes
    if: github.event_name == 'pull_request' && needs.changes.outputs.code == 'true'
    runs-on: blacksmith-4vcpu-windows-2025
    steps:
      - uses: actions/checkout@v3
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '20.x'
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: pnpm --filter create-checkly run test:e2e

  # ---------------------------------------------------------------------------
  # Gates — one aggregate status check per OS. These are the jobs to require in
  # branch protection: a single check for Windows (which fans out into 4 e2e
  # shards) and a single check for Ubuntu, so Windows can be made non-required
  # independently of Ubuntu without touching individual job names.
  # ---------------------------------------------------------------------------
  gate-ubuntu:
    name: CI gate - ubuntu
    if: always()
    needs:
      - changes
      - lint
      - test-ubuntu
      - test-e2e-checkly-ubuntu
      - test-e2e-create-checkly-ubuntu
    runs-on: ubuntu-latest
    steps:
      - name: Verify all Ubuntu jobs succeeded
        run: |
          results="${{ join(needs.*.result, ', ') }}"
          echo "Ubuntu job results: $results"
          # 'skipped' is allowed (e2e jobs are skipped on push-to-main, and
          # every job is skipped for meta-file-only changes); only
          # 'failure'/'cancelled' should fail the gate. `changes` is a
          # dependency so that a failure to compute the path filter fails the
          # gate instead of green-lighting a matrix that never ran.
          if [[ "$results" == *failure* || "$results" == *cancelled* ]]; then
            echo "::error::One or more Ubuntu jobs did not succeed"
            exit 1
          fi
  gate-windows:
    name: CI gate - windows
    if: always()
    needs:
      - changes
      - test-windows
      - test-e2e-checkly-windows
      - test-e2e-create-checkly-windows
    runs-on: ubuntu-latest
    steps:
      - name: Verify all Windows jobs succeeded
        run: |
          results="${{ join(needs.*.result, ', ') }}"
          echo "Windows job results: $results"
          if [[ "$results" == *failure* || "$results" == *cancelled* ]]; then
            echo "::error::One or more Windows jobs did not succeed"
            exit 1
          fi