diff --git a/.github/actions/run-pytest/action.yml b/.github/actions/run-pytest/action.yml new file mode 100644 index 0000000..07dfad3 --- /dev/null +++ b/.github/actions/run-pytest/action.yml @@ -0,0 +1,78 @@ +name: Run pytest with coverage +description: Install deps, verify PostgreSQL, run pytest, upload coverage artifacts + +inputs: + database-url: + description: PostgreSQL connection URL for tests + required: true + artifact-suffix: + description: OS label for artifact names (e.g. ubuntu, macos, windows) + required: true + upload-artifacts: + description: Upload coverage and junit artifacts + required: false + default: "true" + +runs: + using: composite + steps: + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + python-version: "3.13" + + - name: Cache uv + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-uv-test + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + shell: bash + env: + SETUPTOOLS_SCM_WRITE_TO_SOURCE: "1" + run: | + uv venv + uv pip install -r requirements-dev.lock + uv pip install -e . + + - name: Verify PostgreSQL connection + shell: bash + env: + DATABASE_URL: ${{ inputs.database-url }} + SECRET_KEY: for-testing-only + DJANGO_SETTINGS_MODULE: config.test_settings + run: uv run python scripts/verify_postgres_connection.py + + - name: Test with pytest + shell: bash + env: + DATABASE_URL: ${{ inputs.database-url }} + SECRET_KEY: for-testing-only + DJANGO_SETTINGS_MODULE: config.test_settings + run: | + uv run pytest --cov --cov-report=html --cov-fail-under=90 --cov-report=xml --junitxml=junit.xml -v + + - name: Upload HTML coverage report + if: always() && inputs.upload-artifacts == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: coverage-html-${{ inputs.artifact-suffix }} + path: htmlcov/ + retention-days: 30 + + - name: Upload XML coverage report + if: always() && inputs.upload-artifacts == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: coverage-xml-${{ inputs.artifact-suffix }} + path: coverage.xml + + - name: Upload test results + if: always() && inputs.upload-artifacts == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: pytest-results-${{ inputs.artifact-suffix }} + path: junit.xml diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 855ad4d..9964263 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -87,7 +87,7 @@ jobs: run: | uv run python scripts/validate_collector_scaffold.py - test: + test-ubuntu: runs-on: ubuntu-latest timeout-minutes: 15 @@ -114,83 +114,67 @@ jobs: with: fetch-depth: 0 - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - python-version: "3.13" - - - name: Cache uv - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: ~/.cache/uv - key: ${{ runner.os }}-uv-test - restore-keys: | - ${{ runner.os }}-uv- - - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y --no-install-recommends pandoc libleveldb-dev g++ - # Fully pinned tree (requirements-dev.in → requirements-dev.lock). - - name: Install dependencies - env: - SETUPTOOLS_SCM_WRITE_TO_SOURCE: "1" - run: | - uv venv - uv pip install -r requirements-dev.lock - uv pip install -e . + - name: Run pytest + uses: ./.github/actions/run-pytest + with: + database-url: postgres://postgres:postgres@127.0.0.1:5432/postgres + artifact-suffix: ubuntu - # Same DATABASE_URL as pytest; 127.0.0.1 avoids occasional localhost → IPv6 quirks on runners. - - name: Verify PostgreSQL connection - env: - DATABASE_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres - SECRET_KEY: for-testing-only - DJANGO_SETTINGS_MODULE: config.test_settings - run: | - uv run python <<'PY' - import django + test-macos: + runs-on: macos-latest + timeout-minutes: 25 - django.setup() - from django.db import connection + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 - connection.ensure_connection() - with connection.cursor() as cursor: - cursor.execute("SELECT 1") - host = connection.settings_dict.get("HOST") or "" - name = connection.settings_dict.get("NAME") or "" - print(f"PostgreSQL OK (host={host!r}, database={name!r})") - PY + - name: Set up PostgreSQL + uses: ikalnytskyi/action-setup-postgres@c4dda34aae1c821e3a771b68b73b13af3198a7ee # v8 + with: + postgres-version: "16" - - name: Test with pytest - env: - DATABASE_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres - SECRET_KEY: for-testing-only - DJANGO_SETTINGS_MODULE: config.test_settings + - name: Install system dependencies run: | - uv run pytest --cov --cov-report=html --cov-fail-under=90 --cov-report=xml --junitxml=junit.xml -v + brew install pandoc leveldb + echo "CPPFLAGS=-I$(brew --prefix leveldb)/include" >> "$GITHUB_ENV" + echo "LDFLAGS=-L$(brew --prefix leveldb)/lib" >> "$GITHUB_ENV" - - name: Upload HTML coverage report - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + - name: Run pytest + uses: ./.github/actions/run-pytest with: - name: coverage-html - path: htmlcov/ - retention-days: 30 + database-url: postgres://postgres:postgres@127.0.0.1:5432/postgres + artifact-suffix: macos - - name: Upload XML coverage report - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + test-windows: + runs-on: windows-latest + timeout-minutes: 25 + + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + - name: Set up PostgreSQL + uses: ikalnytskyi/action-setup-postgres@c4dda34aae1c821e3a771b68b73b13af3198a7ee # v8 with: - name: coverage-xml - path: coverage.xml + postgres-version: "16" - - name: Upload test results - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + - name: Install system dependencies + run: choco install pandoc -y + + - name: Run pytest + uses: ./.github/actions/run-pytest with: - name: pytest-results - path: junit.xml + database-url: postgres://postgres:postgres@127.0.0.1:5432/postgres + artifact-suffix: windows compose-smoke: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18f9d55..09095c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -164,7 +164,7 @@ Commit the updated `.in` and `.lock` files. Prefer fixing versions over long-liv ## Other guidelines - **Branching:** Create feature branches from `develop`. Open pull requests against `develop`. See [docs/Development_guideline.md](docs/Development_guideline.md). -- **Code style:** Use Python 3.13 and follow Django and project conventions. Use the project’s logging (`logging.getLogger(__name__)`). Before pushing, run **`uv run pyright`** (with dev deps) for the paths covered by **`pyrightconfig.json`**, and ensure CI’s **lint** / **pyright** / **test** / **Security audit** jobs would pass. +- **Code style:** Use Python 3.13 and follow Django and project conventions. Use the project’s logging (`logging.getLogger(__name__)`). Before pushing, run **`uv run pyright`** (with dev deps) for the paths covered by **`pyrightconfig.json`**, and ensure CI’s **lint** / **pyright** / **test-ubuntu** / **test-macos** / **test-windows** / **Security audit** jobs would pass. - **Database:** Use the Django ORM and migrations. Writes only through the service layer as above. - **Docs:** Update this file (and app `services.py` docstrings) when adding new apps or changing the write rules. After changing `services.py` or `core/protocols.py`, run `python scripts/generate_service_docs.py` and commit the updated `docs/service_api/` files. - **Stability:** Pull requests that change `sync_api.__all__`, the `/health/` JSON contract, or management command names used in `config/boost_collector_schedule.yaml` must update [STABILITY.md](STABILITY.md) and [CHANGELOG.md](CHANGELOG.md) when the change is user-visible. diff --git a/README.md b/README.md index bdd9deb..ae86362 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The **`pandoc`** executable must be on your **`PATH`** when you run **`run_boost **When you need it:** Running or debugging the Boost library docs collector without mocks. Many unit tests mock conversion and do not require pandoc on your machine. -**CI:** The GitHub Actions **`test`** job installs pandoc on **`ubuntu-latest`**. The **`lint`** and **`pyright`** jobs do not install it. Developers on macOS or Windows should install pandoc locally if they run integration-style tests or the real collector. +**CI:** The GitHub Actions **`test-ubuntu`**, **`test-macos`**, and **`test-windows`** jobs install pandoc (apt / Homebrew / Chocolatey) before pytest. The **`lint`** and **`pyright`** jobs do not install it. **`compose-smoke`** (Docker stack validation) runs on **`ubuntu-latest`** only. ### Initial setup @@ -172,7 +172,18 @@ python -m pytest --tb=short --cov=. --cov-report=term-missing --cov-fail-under=9 Coverage writes a local **`.coverage`** file (binary data used by `coverage.py`; safe to delete). It is listed in `.gitignore`. -**CI:** [`.github/workflows/actions.yml`](.github/workflows/actions.yml) runs three jobs on pushes/PRs (see the workflow for triggers): **`lint`** (pre-commit on all files), **`pyright`** (static analysis from `pyrightconfig.json`), and **`test`** (pytest with Postgres, `DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/postgres`, `DJANGO_SETTINGS_MODULE=config.test_settings`, coverage, and `--cov-fail-under=90`). The **`test`** job installs **`pandoc`** via apt on Ubuntu; on macOS or Windows, install pandoc yourself if you run the full suite or docs-tracker paths that invoke real conversion (see [System dependencies](#system-dependencies)). +**CI:** [`.github/workflows/actions.yml`](.github/workflows/actions.yml) runs on pushes/PRs (see the workflow for triggers): + +| Job | OS | What it validates | +| --- | --- | --- | +| `test-ubuntu` | Linux | Full pytest + Postgres service container | +| `test-macos` | macOS | Full pytest + native Postgres | +| `test-windows` | Windows | Full pytest + native Postgres (no `plyvel`) | +| `lint` | Linux | pre-commit on all files | +| `pyright` | Linux | Static analysis from `pyrightconfig.json` | +| `compose-smoke` | Linux | Docker Compose stack | + +All three **`test-*`** jobs use `DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/postgres`, `DJANGO_SETTINGS_MODULE=config.test_settings`, coverage, and `--cov-fail-under=90`. Treat failures on these jobs as merge-blocking; add them as required status checks on protected branches if they are not already (see [docs/CODEOWNERS_and_branch_protection.md](docs/CODEOWNERS_and_branch_protection.md)). 6. Run a subset of tests (e.g. one app or one file): diff --git a/docs/CODEOWNERS_and_branch_protection.md b/docs/CODEOWNERS_and_branch_protection.md index 5cd321f..0cf9788 100644 --- a/docs/CODEOWNERS_and_branch_protection.md +++ b/docs/CODEOWNERS_and_branch_protection.md @@ -14,6 +14,16 @@ For each protected branch (for example `main` or `develop`): Without step 4, owners may still appear as suggested reviewers, but merges are not blocked on owner review. +### Required status checks (CI) + +Branch protection on `develop` currently enforces CODEOWNERS review and approvals only—it does **not** require CI jobs to pass before merge. To block merges when tests fail, enable **Require status checks to pass before merging** and add at least: + +- `test-ubuntu` +- `test-macos` +- `test-windows` + +Optionally add `lint`, `pyright`, `compose-smoke`, and jobs from [`.github/workflows/security-audit.yml`](../.github/workflows/security-audit.yml) per team policy. Job names must match the workflow job `id` values exactly. + **Status (`develop`):** Branch protection with **Require review from Code Owners** and **1** required approval was enabled on `cppalliance/boost-data-collector` (verified 2026-05-26). Re-check with: ```bash diff --git a/docs/Celery_test.md b/docs/Celery_test.md index 1a865f4..93a01ba 100644 --- a/docs/Celery_test.md +++ b/docs/Celery_test.md @@ -23,7 +23,7 @@ Open a terminal in the project root and run: celery -A config worker -l info ``` -**Windows:** The project configures the worker to use the `solo` pool on Windows automatically, so you don't get `PermissionError: [WinError 5]`. If you still see that error, run: `celery -A config worker -l info --pool=solo` +**Windows:** The project configures the worker to use the `solo` pool on Windows automatically, so you don't get `PermissionError: [WinError 5]`. If you still see that error, run: `celery -A config worker -l info --pool=solo`. CI runs the full pytest suite on **`windows-latest`** (`test-windows` job in [`.github/workflows/actions.yml`](../.github/workflows/actions.yml)). Leave this running. You should see something like: diff --git a/requirements-dev.lock b/requirements-dev.lock index 15722ee..9e04a1c 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -2,7 +2,7 @@ # uv pip compile requirements-dev.in -o requirements-dev.lock --python-version 3.13 --python-platform linux aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.14.0 +aiohttp==3.14.1 # via # -r requirements.in # discord-py @@ -53,8 +53,9 @@ click-repl==0.3.0 # via celery coverage==7.14.0 # via pytest-cov -cryptography==48.0.0 +cryptography==48.0.1 # via + # -r requirements.in # google-auth # pyjwt discord-py==2.7.1 @@ -159,7 +160,7 @@ pluggy==1.6.0 # via # pytest # pytest-cov -plyvel==1.5.1 +plyvel==1.5.1 ; sys_platform != "win32" # via -r requirements.in portalocker==2.10.1 # via -r requirements.in diff --git a/requirements.in b/requirements.in index 7abe1d1..6b8b9d0 100644 --- a/requirements.in +++ b/requirements.in @@ -3,6 +3,7 @@ # python -m uv pip compile requirements.in -o requirements.lock --python-version 3.13 --python-platform linux # python -m uv pip compile requirements-dev.in -o requirements-dev.lock --python-version 3.13 --python-platform linux # Do not compile on Windows alone: browser-cookie3 pulls pywin32 into the lock and breaks Linux CI. +# After recompiling, confirm plyvel keeps `; sys_platform != "win32"` in both lock files (uv may omit it). # --- Core web / config --- Django>=4.2,<5 @@ -15,9 +16,11 @@ urllib3>=2.0,<3 idna>=3.15,<4 # PYSEC-2026-175..179: ensure patched PyJWT (transitive via PyGithub, redis). PyJWT>=2.13.0,<3 +# GHSA-537c-gmf6-5ccf: fixed in cryptography 48.0.1 (transitive via google-auth, etc.). +cryptography>=48.0.1,<49 discord.py>=2.3.0,<3 -# CVE-2026-34993, CVE-2026-47265: fixed in aiohttp 3.14.0 (transitive via discord.py). -aiohttp>=3.14.0,<4 +# CVE-2026-34993, CVE-2026-47265: fixed in aiohttp 3.14.0; CVE-2026-54273..54280: fixed in 3.14.1. +aiohttp>=3.14.1,<4 python-dateutil>=2.8.0,<3 celery[redis]>=5.3,<6 redis>=5.0,<6 diff --git a/requirements.lock b/requirements.lock index 7c20ab9..0b5b06d 100644 --- a/requirements.lock +++ b/requirements.lock @@ -2,7 +2,7 @@ # uv pip compile requirements.in -o requirements.lock --python-version 3.13 --python-platform linux aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.14.0 +aiohttp==3.14.1 # via # -r requirements.in # discord-py @@ -48,8 +48,9 @@ click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -cryptography==48.0.0 +cryptography==48.0.1 # via + # -r requirements.in # google-auth # pyjwt discord-py==2.7.1 @@ -106,7 +107,7 @@ pinecone==6.0.2 # via -r requirements.in pinecone-plugin-interface==0.0.7 # via pinecone -plyvel==1.5.1 +plyvel==1.5.1 ; sys_platform != "win32" # via -r requirements.in portalocker==2.10.1 # via -r requirements.in diff --git a/scripts/verify_postgres_connection.py b/scripts/verify_postgres_connection.py new file mode 100644 index 0000000..dd3b3d0 --- /dev/null +++ b/scripts/verify_postgres_connection.py @@ -0,0 +1,12 @@ +"""Verify Django can connect to PostgreSQL (used by CI on all platforms).""" + +import django +from django.db import connection + +django.setup() +connection.ensure_connection() +with connection.cursor() as cursor: + cursor.execute("SELECT 1") +host = connection.settings_dict.get("HOST") or "" +name = connection.settings_dict.get("NAME") or "" +print(f"PostgreSQL OK (host={host!r}, database={name!r})")