diff --git a/.github/workflows/cicd-2-main-branch.yaml b/.github/workflows/cicd-2-main-branch.yaml index 40e80b33..94f14e19 100644 --- a/.github/workflows/cicd-2-main-branch.yaml +++ b/.github/workflows/cicd-2-main-branch.yaml @@ -33,6 +33,39 @@ jobs: uses: ./.github/workflows/stage-2-test.yaml secrets: inherit + # Publish the test report (produced by the test stage) to GitHub Pages so + # assurance reviewers have a stable, per-build link to the latest results. + # Runs even if tests fail, so a failing report is still published. + publish-test-report: + name: Publish test report + needs: test-stage + if: always() && needs.test-stage.result != 'skipped' + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Download test report + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: test-report + path: reports + + - name: Configure Pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: reports + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 + # Resolve the latest release tag once so downstream jobs can consume it. resolve: name: Resolve deployment targets diff --git a/.github/workflows/stage-2-test.yaml b/.github/workflows/stage-2-test.yaml index ae0e769c..b2d320c4 100644 --- a/.github/workflows/stage-2-test.yaml +++ b/.github/workflows/stage-2-test.yaml @@ -36,7 +36,7 @@ jobs: run: uv sync --all-extras - name: Run tests - run: uv run pytest + run: uv run pytest --html=reports/index.html --self-contained-html - name: Upload coverage reports uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 @@ -47,6 +47,14 @@ jobs: name: codecov-umbrella fail_ci_if_error: false + - name: Upload test report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() # capture the report even when tests fail + with: + name: test-report + path: reports/ + if-no-files-found: error + test-lint: name: Lint and type checking runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 7a17fe53..b17c3221 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ build/ .coverage.* htmlcov/ coverage.xml +reports/ .tox/ .hypothesis/ diff --git a/docs/testing-statement.md b/docs/testing-statement.md new file mode 100644 index 00000000..e19d1f55 --- /dev/null +++ b/docs/testing-statement.md @@ -0,0 +1,77 @@ +# Rubie Gateway — Testing Statement + +**Purpose:** Describe how the Rubie Gateway is tested, what is and is not covered by automated testing, and where test evidence can be found. +**Status:** Living document — figures reflect the state of `main` and should be refreshed when materially changed. + +--- + +## 1. What the gateway does (context for scope) + +The Rubie Gateway is a Python application that runs on a VM inside an NHS Trust network and bridges the on-site imaging modality and the cloud-based Rubie service. Its responsibilities are: + +- **MWL (Modality Worklist) SCP** — receives worklist create/update commands from Rubie (via Azure Relay), and serves them to the modality over DICOM C-FIND. +- **PACS SCP** — receives acquired images from the modality over DICOM C-STORE, and answers DICOM C-ECHO connectivity checks. +- **Upload service** — forwards stored images to the Rubie cloud API. +- **Relay listener** — maintains the Azure Relay connection over which Rubie sends commands. + +Testing is scoped to the behaviour of this application. The wider distributed system (Rubie cloud, NBSS, the Trust PACS, the physical modality) is out of scope for automated testing here. + +## 2. Testing approach + +The strategy is recorded in [ADR-002: Testing strategy](./adr/ADR-002_Testing_strategy.md). In summary, the suite is layered: + +| Layer | What it exercises | Dependencies | +| --- | --- | --- | +| **Unit** | Individual functions and classes in isolation | Mocked | +| **Integration** | Components working together, including the gateway server running in a background thread | Real SQLite database, real filesystem | +| **End-to-end (in-process)** | A full relay-command → MWL → C-STORE → upload flow | Real SQLite + filesystem; external network boundaries emulated | + +Because it is impractical to stand up the entire Rubie + PACS + modality ecosystem for automated testing, the suite uses **emulation** at the system boundaries: crafted DICOM datasets and a modality emulator stand in for real imaging hardware, and the Azure Relay transport is mocked so command-processing logic can be tested deterministically. + +## 3. What is covered + +Automated tests cover the core DICOM and messaging behaviour, including: + +- **MWL**: C-FIND worklist queries (including patient-name search), worklist item creation, and status transitions. +- **PACS**: C-STORE image receipt and metadata persistence, handling of already-stored SOP instances, and C-ECHO verification (both PACS and MWL application entities). +- **Relay listener**: processing of inbound action commands and dispatch to the correct handler. +- **Upload**: forwarding of stored instances to the Rubie API. +- **Modality emulator**: worklist querying and image generation used to drive end-to-end tests. + +**Current figures (refresh on material change):** + +- **Over 200 automated tests** (194 unit, 19 integration), all passing on `main`. +- **Approximately 90% line coverage** of application code (`src/`), measured by `pytest-cov` and reported to Codecov on every run. + +## 4. Automated quality gates (CI) + +Every pull request and every merge to `main` runs the following in GitHub Actions; all must pass before code is deployable: + +| Stage | Check | Tool | +| --- | --- | --- | +| Commit | Secret scanning across branch history | gitleaks | +| Test | Full unit + integration suite | pytest | +| Test | Line-coverage measurement and reporting | pytest-cov / Codecov | +| Test | Linting and formatting | ruff | +| Test | Static type checking | pyright | +| Scheduled | Static application security testing (SAST) | CodeQL (weekly) | + +## 5. Test evidence + +- **Per-run evidence:** the GitHub Actions run for each PR and each `main` build records the full test output, pass/fail status, and coverage upload. These are retained against the commit and pull request. +- **Coverage trend:** reported to Codecov, giving per-file coverage and change-over-time. +- **Local reproduction:** the full suite is `make test`; subsets are `make test-unit` and `make test-integration`. Coverage HTML is written to `htmlcov/`. +- **Published report:** a human-readable test report is published to GitHub Pages on every `main` build and is available at . This gives assurance reviewers a stable link to the latest results without navigating CI logs, and is refreshed automatically per build. + +## 6. What is NOT covered by automated testing + +| Not automatically tested | Why | Compensating assurance | +| --- | --- | --- | +| **Real modality hardware** | No physical modality in CI | Emulated with crafted DICOM and the modality emulator; validated against real Hologic kit during site dry runs. | +| **Live Azure Relay / Managed Identity auth** | No Azure resources in CI; the transport is mocked | Credential and connection logic is exercised by deployments to dev/pre-prod and by post-deployment smoke tests. | +| **PowerShell deployment scripts** (`deploy.ps1`, `rollback.ps1`) | Windows blue/green cutover, NSSM service install and rollback are not unit-testable in this suite | Exercised by real deployments to non-production environments with smoke tests; Bash helper scripts are covered by ShellCheck. | +| **The end-to-end distributed system** (Rubie cloud, NBSS, Trust PACS) | Impractical to assemble for automated testing (ADR-002) | Emulation at boundaries; manual/clinical validation via site dry runs. | + +## 7. Maintaining this statement + +This statement should be reviewed when the testing strategy changes materially — for example a new test layer, a significant coverage change, a new CI gate, or a change to what is emulated versus tested against real dependencies. The figures in §3 are point-in-time and can be regenerated with `make test`. diff --git a/pyproject.toml b/pyproject.toml index bd046288..e3b09acc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ dev = [ "pytest>=9.0.3,<10", "pytest-cov>=7.0.0,<8", + "pytest-html>=4.1,<5", "ruff>=0.14.1,<0.16", "pyright>=1.1.390", "ipdb>=0.13.13,<0.14", diff --git a/uv.lock b/uv.lock index 8a0b679f..58868366 100644 --- a/uv.lock +++ b/uv.lock @@ -506,6 +506,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "pytest-html" }, { name = "python-semantic-release" }, { name = "ruff" }, ] @@ -532,6 +533,7 @@ dev = [ { name = "pytest", specifier = ">=9.0.3,<10" }, { name = "pytest-asyncio", specifier = ">=1.3.0,<1.4.0" }, { name = "pytest-cov", specifier = ">=7.0.0,<8" }, + { name = "pytest-html", specifier = ">=4.1,<5" }, { name = "python-semantic-release", specifier = ">=10.5.3" }, { name = "ruff", specifier = ">=0.14.1,<0.16" }, ] @@ -1247,6 +1249,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, ] +[[package]] +name = "pytest-html" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "pytest" }, + { name = "pytest-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/08/2076aa09507e51c1119d16a84c6307354d16270558f1a44fc9a2c99fdf1d/pytest_html-4.2.0.tar.gz", hash = "sha256:b6a88cba507500d8709959201e2e757d3941e859fd17cfd4ed87b16fc0c67912", size = 108634, upload-time = "2026-01-19T11:25:26.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/47/07046e0acedc12fe2bae79cf6c73ad67f51ae9d67df64d06b0f3eac73d36/pytest_html-4.2.0-py3-none-any.whl", hash = "sha256:ff5caf3e17a974008e5816edda61168e6c3da442b078a44f8744865862a85636", size = 23801, upload-time = "2026-01-19T11:25:25.008Z" }, +] + +[[package]] +name = "pytest-metadata" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/85/8c969f8bec4e559f8f2b958a15229a35495f5b4ce499f6b865eac54b878d/pytest_metadata-3.1.1.tar.gz", hash = "sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8", size = 9952, upload-time = "2024-02-12T19:38:44.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b", size = 11428, upload-time = "2024-02-12T19:38:42.531Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2"