test(recorded): add replay harness

Author: Pouyanpi

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all test tests test_watch test_coverage test_profile docs docs-strict docs-serve docs-update-cards docs-check-cards docs-watch-cards pre_commit help
+.PHONY: all test tests test_watch test_coverage test_profile record-tests docs docs-strict docs-serve docs-update-cards docs-check-cards docs-watch-cards pre_commit help
 
 # Default target executed when no specific target is provided to make.
 all: help
@@ -21,6 +21,15 @@ test_coverage:
 test_profile:
 	poetry run pytest -vv tests/ --profile-svg
 
+# Refresh recorded-test cassettes against live providers, fill snapshots, then verify
+# the offline replay. Requires real provider credentials in the environment
+# (e.g. OPENAI_API_KEY, NVIDIA_API_KEY). Fake cassettes are excluded from recording.
+RECORDED_TESTS ?= tests/recorded
+record-tests:
+	poetry run pytest $(RECORDED_TESTS) --record-mode=all -m "not fake_cassette"
+	poetry run pytest $(RECORDED_TESTS) --block-network --inline-snapshot=create
+	poetry run pytest $(RECORDED_TESTS) --block-network
+
 docs:
 	poetry run sphinx-build -b html docs _build/docs
 

poetry.lock

@@ -160,6 +160,7 @@ pytest = ">=7.2.2,<9.0.0"
 pytest-asyncio = ">=0.21.0, <1.0.0"
 pytest-cov = ">=4.1.0"
 pytest-httpx = ">=0.22.0"
+pytest-recording = "^0.13.4"
 streamlit = ">=1.37.0"
 tox = "^4.23.2"
 pytest-profiling = "^1.7.0"
@@ -184,6 +185,8 @@ langchain-core = ">=0.2.14,<2.0.0"
 langchain-community = ">=0.2.5,<2.0.0"
 langchain-openai = ">=0.1.0"
 langchain-nvidia-ai-endpoints = ">=0.2.0"
+inline-snapshot = "^0.33.0"
+dirty-equals = "^0.11"
 
 # Directories in which to run Pyright type-checking
 [tool.pyright]
@@ -232,6 +235,9 @@ log-level = "DEBUG"
 # phase, which will cause tests to fail or "magically" ignored.
 log_cli = "False"
 
+[tool.inline-snapshot]
+format-command = "ruff format --stdin-filename {filename} -"
+
 [build-system]
 requires = ["poetry-core>=1.0.0,<2.0.0"]
 build-backend = "poetry.core.masonry.api"

pyproject.toml

@@ -10,6 +10,12 @@ log_cli = False
 
 asyncio_default_fixture_loop_scope = function
 
+markers =
+    recorded: deterministic cassette replay tests
+    live: tests that intentionally hit external services
+    vcr: marker used by pytest-recording
+    fake_cassette: cassette is hand-authored; refresh workflow must skip these tests
+
 testpaths =
     tests
     docs/colang-2/examples

pytest.ini

@@ -0,0 +1,133 @@
+# Recorded Tests
+
+Recorded tests replay provider traffic through pytest-recording cassettes and must run without live network access by default.
+
+## Adding a test
+
+Markers are applied once per module via `pytestmark`; do not stack `@pytest.mark.recorded` / `vcr` / `asyncio` on each test. Use a module-level list, and fold in `vcr`/`asyncio` only when every test in the module needs them:
+
+```python
+pytestmark = [pytest.mark.recorded, pytest.mark.vcr, pytest.mark.asyncio]
+
+
+async def test_my_case(openai_api_key):
+    rails = LLMRails(load_config(OPENAI_BASELINE_CONFIG), verbose=False)
+    result = await rails.generate_async(prompt="...")
+    assert result == snapshot()
+```
+
+Request credentials as fixture parameters (`openai_api_key`, `nvidia_api_key`) rather than calling `request.getfixturevalue(...)`. In modules that mix sync/async tests or vcr/non-vcr tests, keep only `recorded` in `pytestmark` and apply `@pytest.mark.vcr` / `@pytest.mark.asyncio` per test. Then record once with credentials, fill the snapshot offline, and verify the replay:
+
+```bash
+OPENAI_API_KEY=... poetry run pytest path::test_my_case --record-mode=all
+poetry run pytest path::test_my_case --block-network --inline-snapshot=create
+poetry run pytest path::test_my_case --block-network
+```
+
+## Negative paths
+
+This suite owns **pipeline-level** failures (how `LLMRails` behaves when a model call
+fails) and **public-API input validation**. Client/wire-level conditions (status code to
+exception mapping, retries, SSE, malformed bodies) belong in `tests/llm/clients/`, which
+covers them with `httpx.MockTransport` + JSON fixtures; do not duplicate them here.
+
+Prefer mechanisms in this order:
+
+1. **Recordable real error** (refreshable cassette). A nonexistent model name yields a real,
+   deterministic 404, so error paths record and refresh like any happy path. Use a config
+   whose model is invalid (see `OPENAI_INVALID_MODEL_CONFIG`,
+   `CONTENT_SAFETY_INVALID_MODEL_CONFIG`).
+2. **Pure runtime** `pytest.raises` for input validation (no cassette, no transport).
+3. **Fake cassette** (`@pytest.mark.fake_cassette`) only as a last resort, for a synthetic
+   response that must flow through the full pipeline and cannot be reproduced by 1 or 2.
+
+Observed behavior these tests pin: a failing model call (main *or* a rail's own model)
+propagates as `LLMCallException` — a safety-model failure does not let content through
+silently. Name negative tests `test_<surface>_<failure>_<behavior>` with the suffixes
+`_raises` / `_fails_closed` / `_invalid_*` so they are greppable
+(`pytest -k "raises or invalid"`), and co-locate each with its happy-path sibling module.
+
+## Replay
+
+```bash
+poetry run pytest tests/recorded --block-network -v --durations=10
+```
+
+Focused rails replay:
+
+```bash
+poetry run pytest tests/recorded/rails/public_api --block-network -v
+poetry run pytest tests/recorded/rails/library --block-network -v
+```
+
+Replay mode installs dummy API keys from `tests/recorded/utils.py`. A cassette miss with `--block-network` is a test failure.
+
+## Refresh
+
+Refresh only in a trusted environment with real provider credentials. The full
+record -> fill-snapshots -> verify loop is wrapped in a make target:
+
+```bash
+OPENAI_API_KEY=... NVIDIA_API_KEY=... make record-tests
+```
+
+Or run the recording step alone:
+
+```bash
+poetry run pytest tests/recorded --record-mode=all -m "not fake_cassette" -v
+```
+
+For a focused rewrite:
+
+```bash
+poetry run pytest tests/recorded/rails/public_api/test_generate.py::test_openai_generate_async_public_contract --record-mode=rewrite -v
+```
+
+The refresh workflow uploads cassettes as artifacts for review and does not commit them.
+
+## Cassettes
+
+Rails tests use pytest-recording's default names:
+
+```text
+tests/recorded/rails/<suite>/cassettes/<test_module>/<test_name>.yaml
+```
+
+Parameterized tests include the parameter id in the cassette filename. Every test (rails and clients) uses this default naming; do not add `@pytest.mark.default_cassette(...)`.
+
+JSON request and response bodies are stored as `parsed_body` and rehydrated by `ReadableYamlSerializer` during replay. SSE responses also use parseable `parsed_body` events.
+
+At serialize time the bodies are normalized to ASCII (smart quotes, en/em dashes, the hyphen family, and ellipsis are folded, then NFKC) so cassettes and inline snapshots stay stable and portable. Response headers are dropped by exact name and by prefix (`x-`, `cf-`, `openai-`); `tests/recorded/sanitization.py` holds the `ALLOWED_HEADERS` exceptions that must survive the prefix sweep (currently `content-type`).
+
+Inspect a cassette:
+
+```bash
+poetry run python -m tests.recorded.inspect_cassette tests/recorded/rails/public_api/cassettes/test_stream/test_openai_stream_async_public_contract.yaml
+```
+
+## Snapshots
+
+Rails replay outputs are pinned with inline snapshots after normalization. Create or fix snapshots with:
+
+```bash
+poetry run pytest tests/recorded/rails --block-network --inline-snapshot=create
+poetry run pytest tests/recorded/rails --block-network --inline-snapshot=fix
+poetry run pytest tests/recorded/rails --block-network --inline-snapshot=review
+```
+
+Snapshot formatting uses `ruff format` through `[tool.inline-snapshot]` in `pyproject.toml`.
+
+Volatile response fields (ids, timestamps, fingerprints) are scrubbed to fixed sentinels in the cassette, so snapshots assert them directly without needing loose matchers.
+
+## Fake Outputs
+
+Prefer `FakeLLMModel` when a test needs the main model to emit a specific output and provider-backed rail/task calls can still replay from VCR. This keeps the test refreshable.
+
+Use a fake cassette only when runtime injection cannot model the behavior clearly, such as a provider stream/error path. Fake cassettes must:
+
+- live under a `cassettes/**/fake/` directory,
+- use `@pytest.mark.fake_cassette`,
+- be excluded from refresh with `-m "not fake_cassette"`,
+- include YAML header metadata with `reason`, `frozen_fields`, and `fake_llm_model_considered`.
+
+The fake-cassette metadata validator is in `tests/recorded/fake_cassettes.py`.

tests/recorded/README.md

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.