refactor(tests): split suites by execution level and speed up CI #5928
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "main" | |
| tags: | |
| - "v*" | |
| merge_group: | |
| pull_request: | |
| branches: | |
| - "*" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| linting: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true | |
| - name: Install dependencies | |
| run: uv sync --locked | |
| - name: Run Ruff | |
| run: uv run --frozen ruff check . | |
| type-checking: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true | |
| - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| name: Cache mypy cache | |
| with: | |
| path: ./.mypy_cache | |
| key: mypy-${{ hashFiles('**/*.py', 'pyproject.toml') }} | |
| restore-keys: | | |
| mypy- | |
| - name: Install dependencies | |
| run: uv sync --locked | |
| - name: Run mypy type checking | |
| run: uv run --frozen mypy langfuse --no-error-summary | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| LANGFUSE_BASE_URL: "http://localhost:3000" | |
| LANGFUSE_PUBLIC_KEY: "pk-lf-test" | |
| LANGFUSE_SECRET_KEY: "sk-lf-test" | |
| OPENAI_API_KEY: "test-openai-key" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| name: Unit tests on Python ${{ matrix.python-version }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: "0.11.2" | |
| python-version: ${{ matrix.python-version }} | |
| enable-cache: true | |
| - name: Check Python version | |
| run: python --version | |
| - name: Install the project dependencies | |
| run: uv sync --locked | |
| - name: Run the automated tests | |
| run: | | |
| python --version | |
| uv run --frozen pytest -n auto --dist worksteal -s -v --log-cli-level=INFO tests/unit | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| LANGFUSE_BASE_URL: "http://localhost:3000" | |
| LANGFUSE_PUBLIC_KEY: "pk-lf-1234567890" | |
| LANGFUSE_SECRET_KEY: "sk-lf-1234567890" | |
| LANGFUSE_E2E_READ_TIMEOUT_SECONDS: "60" | |
| LANGFUSE_E2E_READ_INTERVAL_SECONDS: "0.5" | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| # SERPAPI_API_KEY: ${{ secrets.SERPAPI_API_KEY }} | |
| HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| name: E2E tests on Python 3.13 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true | |
| - name: Install the project dependencies | |
| run: uv sync --locked | |
| - name: Check uv Python version | |
| run: uv run --frozen python --version | |
| - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5 | |
| with: | |
| version: 10.33.0 | |
| - name: Clone langfuse server | |
| run: | | |
| git clone https://github.com/langfuse/langfuse.git ./langfuse-server && echo $(cd ./langfuse-server && git rev-parse HEAD) | |
| - name: Setup node (for langfuse server) | |
| uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 | |
| with: | |
| node-version: 24 | |
| - name: Cache langfuse server dependencies | |
| uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: ./langfuse-server/node_modules | |
| key: | | |
| langfuse-server-${{ hashFiles('./langfuse-server/package-lock.json') }} | |
| langfuse-server- | |
| - name: Run langfuse server | |
| run: | | |
| cd ./langfuse-server | |
| echo "::group::Run langfuse server" | |
| TELEMETRY_ENABLED=false docker compose up -d postgres | |
| echo "::endgroup::" | |
| echo "::group::Logs from langfuse server" | |
| TELEMETRY_ENABLED=false docker compose logs | |
| echo "::endgroup::" | |
| echo "::group::Install dependencies (necessary to run seeder)" | |
| pnpm i | |
| echo "::endgroup::" | |
| echo "::group::Seed db" | |
| cp .env.dev.example .env | |
| pnpm run db:migrate | |
| pnpm run db:seed | |
| echo "::endgroup::" | |
| rm -rf .env | |
| echo "::group::Run server" | |
| TELEMETRY_ENABLED=false \ | |
| LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090 \ | |
| LANGFUSE_INGESTION_QUEUE_DELAY_MS=10 \ | |
| LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10 \ | |
| LANGFUSE_EXPERIMENT_INSERT_INTO_EVENTS_TABLE=true \ | |
| QUEUE_CONSUMER_EVENT_PROPAGATION_QUEUE_IS_ENABLED=true \ | |
| LANGFUSE_ENABLE_EVENTS_TABLE_V2_APIS=true \ | |
| LANGFUSE_ENABLE_EVENTS_TABLE_OBSERVATIONS=true \ | |
| docker compose up -d | |
| echo "::endgroup::" | |
| - name: Health check for langfuse server | |
| run: | | |
| echo "Checking if the langfuse server is up..." | |
| retry_count=0 | |
| max_retries=10 | |
| until curl --output /dev/null --silent --head --fail http://localhost:3000/api/public/health | |
| do | |
| retry_count=`expr $retry_count + 1` | |
| echo "Attempt $retry_count of $max_retries..." | |
| if [ $retry_count -ge $max_retries ]; then | |
| echo "Langfuse server did not respond in time. Printing logs..." | |
| docker logs langfuse-server-langfuse-web-1 | |
| echo "Failing the step..." | |
| exit 1 | |
| fi | |
| sleep 5 | |
| done | |
| echo "Langfuse server is up and running!" | |
| - name: Run the end-to-end tests | |
| run: | | |
| uv run --frozen python --version | |
| uv run --frozen pytest -n 2 --dist loadfile -s -v --log-cli-level=INFO tests/e2e | |
| all-tests-passed: | |
| # This allows us to have a branch protection rule for tests and deploys with matrix | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, e2e-tests, linting, type-checking] | |
| if: always() | |
| steps: | |
| - name: Successful deploy | |
| if: ${{ !(contains(needs.*.result, 'failure')) }} | |
| run: exit 0 | |
| - name: Failing deploy | |
| if: ${{ contains(needs.*.result, 'failure') }} | |
| run: exit 1 |