refactor(tests): split suites by execution level and speed up CI #5935
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "main" | |
| tags: | |
| - "v*" | |
| merge_group: | |
| pull_request: | |
| branches: | |
| - "*" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| linting: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true | |
| - name: Install dependencies | |
| run: uv sync --locked | |
| - name: Run Ruff | |
| run: uv run --frozen ruff check . | |
| type-checking: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true | |
| - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| name: Cache mypy cache | |
| with: | |
| path: ./.mypy_cache | |
| key: mypy-${{ hashFiles('**/*.py', 'pyproject.toml') }} | |
| restore-keys: | | |
| mypy- | |
| - name: Install dependencies | |
| run: uv sync --locked | |
| - name: Run mypy type checking | |
| run: uv run --frozen mypy langfuse --no-error-summary | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| LANGFUSE_BASE_URL: "http://localhost:3000" | |
| LANGFUSE_PUBLIC_KEY: "pk-lf-test" | |
| LANGFUSE_SECRET_KEY: "sk-lf-test" | |
| OPENAI_API_KEY: "test-openai-key" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| name: Unit tests on Python ${{ matrix.python-version }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: "0.11.2" | |
| python-version: ${{ matrix.python-version }} | |
| enable-cache: true | |
| - name: Check Python version | |
| run: python --version | |
| - name: Install the project dependencies | |
| run: uv sync --locked | |
| - name: Run the automated tests | |
| run: | | |
| python --version | |
| uv run --frozen pytest -n auto --dist worksteal -s -v --log-cli-level=INFO tests/unit | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - shard-name: core | |
| test-root: tests/e2e | |
| parallel-marker: "e2e_core and not serial_e2e" | |
| serial-marker: "e2e_core and serial_e2e" | |
| - shard-name: data | |
| test-root: tests/e2e | |
| parallel-marker: "e2e_data and not serial_e2e" | |
| serial-marker: "e2e_data and serial_e2e" | |
| - shard-name: live-provider | |
| test-root: tests/live_provider | |
| parallel-marker: "live_provider" | |
| serial-marker: "" | |
| env: | |
| LANGFUSE_BASE_URL: "http://localhost:3000" | |
| LANGFUSE_PUBLIC_KEY: "pk-lf-1234567890" | |
| LANGFUSE_SECRET_KEY: "sk-lf-1234567890" | |
| LANGFUSE_INIT_ORG_ID: "0c6c96f4-0ca0-4f16-92a8-6dd7d7c6a501" | |
| LANGFUSE_INIT_ORG_NAME: "SDK Test Org" | |
| LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" | |
| LANGFUSE_INIT_PROJECT_NAME: "SDK Test Project" | |
| LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890" | |
| LANGFUSE_INIT_PROJECT_SECRET_KEY: "sk-lf-1234567890" | |
| LANGFUSE_INIT_USER_EMAIL: "sdk-tests@langfuse.local" | |
| LANGFUSE_INIT_USER_NAME: "SDK Tests" | |
| LANGFUSE_INIT_USER_PASSWORD: "langfuse-ci-password" | |
| LANGFUSE_E2E_READ_TIMEOUT_SECONDS: "60" | |
| LANGFUSE_E2E_READ_INTERVAL_SECONDS: "0.5" | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| # SERPAPI_API_KEY: ${{ secrets.SERPAPI_API_KEY }} | |
| HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| name: E2E ${{ matrix.shard-name }} tests on Python 3.13 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true | |
| - name: Install the project dependencies | |
| run: uv sync --locked | |
| - name: Check uv Python version | |
| run: uv run --frozen python --version | |
| - name: Prepare langfuse server compose | |
| run: | | |
| mkdir -p ./langfuse-server | |
| LANGFUSE_SERVER_SHA="$(git ls-remote https://github.com/langfuse/langfuse.git HEAD | cut -f1)" | |
| curl -fsSL "https://raw.githubusercontent.com/langfuse/langfuse/${LANGFUSE_SERVER_SHA}/docker-compose.yml" \ | |
| -o ./langfuse-server/docker-compose.yml | |
| echo "${LANGFUSE_SERVER_SHA}" | |
| - name: Run langfuse server | |
| run: | | |
| cd ./langfuse-server | |
| echo "::group::Start langfuse server" | |
| TELEMETRY_ENABLED=false \ | |
| NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=true \ | |
| LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090 \ | |
| LANGFUSE_INGESTION_QUEUE_DELAY_MS=10 \ | |
| LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10 \ | |
| LANGFUSE_EXPERIMENT_INSERT_INTO_EVENTS_TABLE=true \ | |
| QUEUE_CONSUMER_EVENT_PROPAGATION_QUEUE_IS_ENABLED=true \ | |
| LANGFUSE_ENABLE_EVENTS_TABLE_V2_APIS=true \ | |
| LANGFUSE_ENABLE_EVENTS_TABLE_OBSERVATIONS=true \ | |
| docker compose up -d | |
| echo "::endgroup::" | |
| - name: Health check for langfuse server | |
| run: | | |
| echo "Checking if the langfuse server is up..." | |
| retry_count=0 | |
| max_retries=20 | |
| until curl --output /dev/null --silent --head --fail http://localhost:3000/api/public/health && \ | |
| uv run --frozen python -c "from langfuse import Langfuse; client = Langfuse(); project_id = client._get_project_id(); assert project_id == '7a88fb47-b4e2-43b8-a06c-a5ce950dc53a', project_id; print(project_id)" | |
| do | |
| retry_count=`expr $retry_count + 1` | |
| echo "Attempt $retry_count of $max_retries..." | |
| if [ $retry_count -ge $max_retries ]; then | |
| echo "Langfuse server did not respond in time. Printing logs..." | |
| (cd ./langfuse-server && docker compose ps) | |
| (cd ./langfuse-server && docker compose logs langfuse-web langfuse-worker) | |
| echo "Failing the step..." | |
| exit 1 | |
| fi | |
| sleep 5 | |
| done | |
| echo "Langfuse server is up and running!" | |
| - name: Run the end-to-end tests | |
| run: | | |
| uv run --frozen python --version | |
| uv run --frozen pytest -n 4 --dist worksteal -s -v --log-cli-level=INFO ${{ matrix.test-root }} -m "${{ matrix.parallel-marker }}" | |
| - name: Run serial end-to-end tests | |
| if: ${{ matrix.serial-marker != '' }} | |
| run: | | |
| uv run --frozen pytest -s -v --log-cli-level=INFO ${{ matrix.test-root }} -m "${{ matrix.serial-marker }}" | |
| all-tests-passed: | |
| # This allows us to have a branch protection rule for tests and deploys with matrix | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, e2e-tests, linting, type-checking] | |
| if: always() | |
| steps: | |
| - name: Successful deploy | |
| if: ${{ !(contains(needs.*.result, 'failure')) }} | |
| run: exit 0 | |
| - name: Failing deploy | |
| if: ${{ contains(needs.*.result, 'failure') }} | |
| run: exit 1 |