diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml new file mode 100644 index 0000000..9a6c6d8 --- /dev/null +++ b/.cargo/mutants.toml @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: MPL-2.0 +# +# Configuration for cargo-mutants — mutation testing. +# +# Usage: +# cargo mutants # run on the whole workspace +# cargo mutants --in-diff # only mutate code changed vs +# cargo mutants -p verisim-drift # focus on a single crate +# cargo mutants --shard 0/4 # parallel sharding +# +# What mutation testing does: it inserts small bug-like changes into your +# code (e.g. `>` → `>=`, `+` → `-`, `true` → `false`) and runs the test +# suite. A mutant that survives means your tests don't distinguish the +# buggy version from the original — you have a coverage gap that line/branch +# metrics can't see. +# +# Target mutation score: 75-85% on core invariants (drift, normaliser, +# octad). HTTP glue and trace setup are exempt because mutating them +# produces uninteresting survivors. + +# --------------------------------------------------------------------------- +# Scope +# --------------------------------------------------------------------------- + +# Skip generated code, vendored protos, and crates whose value lies in +# their I/O surface rather than their logic. +exclude_globs = [ + # Generated protobuf code — mutating it tests nothing real + "rust-core/verisim-api/src/proto/**", + # Repl/binary entry points — mutations here exercise rustyline/atty + # rather than VeriSimDB logic + "rust-core/verisim-repl/src/main.rs", + "rust-core/verisim-api/src/main.rs", + # Fuzz harnesses — mutations would just make them not fuzz + "fuzz/**", + "rust-core/fuzz/**", + # Benchmarks — mutations break bench harness, not project logic + "benches/**", + # Build scripts + "**/build.rs", +] + +# Match cargo-mutants' default mutator set. If a specific mutator produces +# too much noise (e.g. constant-replacement on cost-model coefficients that +# are themselves heuristic), pin operators here: +# operators = ["binop", "logop", "bool_lit", "return"] + +# --------------------------------------------------------------------------- +# Test execution +# --------------------------------------------------------------------------- + +# Use cargo nextest if available — significantly faster than cargo test for +# large mutation runs. Falls back to cargo test automatically. +# test_tool = "nextest" + +# Per-mutant timeout multiplier — generous to avoid false-positive timeouts +# on the slower spatial/temporal tests. +timeout_multiplier = 3.0 + +# Run the *baseline* (no mutation) once before applying mutants; if it fails +# the run aborts early with a clear "your tests are broken before any mutation" +# message rather than blaming the mutator. +# (cargo-mutants does this by default; documented here for clarity.) + +# --------------------------------------------------------------------------- +# Per-crate strategy +# --------------------------------------------------------------------------- +# +# Mutation testing is most valuable where invariants are encoded in pure +# functions: drift calculation, hash-chain integrity, constraint +# evaluation, k-nearest sorting. Less valuable for HTTP routing, error +# message strings, and IPC glue. +# +# Priority crates (mutation score target ≥80%): +# - verisim-drift (drift score arithmetic, threshold policies) +# - verisim-normalizer (regeneration strategies, conflict resolution) +# - verisim-spatial (haversine distance, R-tree search) +# - verisim-provenance (hash-chain verification) +# - verisim-semantic (constraint validation, ZKP primitives) +# - verisim-octad (transaction state machine, WAL replay) +# +# Best-effort crates (no enforced floor): +# - verisim-api (HTTP handlers — mutations land in plumbing) +# - verisim-repl (CLI surface — see exclusions above) +# - verisim-graph (in-memory dispatcher, thin layer) +# +# For PR CI, run `cargo mutants --in-diff origin/main` to only mutate +# changed lines. This keeps per-PR runtime under ~5 minutes for typical +# changesets. diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 6cb8408..888d3a5 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -113,6 +113,39 @@ jobs: - name: Enforce minimum coverage run: cargo llvm-cov --workspace --summary-only --fail-under-lines 60 + mutants: + # Mutation testing — runs only on workflow_dispatch + scheduled because + # full-workspace mutants can take >20 min. For PRs the recommended + # invocation is `cargo mutants --in-diff origin/main` which we can wire + # into a separate pr-mutants.yml once the suite stabilises. + name: cargo-mutants (on-demand) + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@v2 + with: + tool: cargo-mutants + - name: Run mutation tests on core invariant crates + run: | + cargo mutants \ + -p verisim-drift \ + -p verisim-normalizer \ + -p verisim-spatial \ + -p verisim-provenance \ + -p verisim-semantic \ + -p verisim-octad \ + --no-shuffle \ + --output mutants-out/ + continue-on-error: true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: mutants-report + path: mutants-out/ + fuzz-compile: name: fuzz targets compile runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 01c5841..6cb60bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1637,6 +1637,17 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -2749,6 +2760,12 @@ dependencies = [ "embedded-hal 1.0.0", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -4049,6 +4066,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "insta" +version = "1.47.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +dependencies = [ + "console", + "once_cell", + "serde", + "similar", + "tempfile", +] + [[package]] name = "instability" version = "0.3.11" @@ -6986,6 +7016,12 @@ dependencies = [ "quote", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "siphasher" version = "1.0.2" @@ -8602,6 +8638,7 @@ name = "verisim-planner" version = "0.1.0" dependencies = [ "chrono", + "insta", "proptest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index bb8dfd9..7dca7a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,7 @@ axum-server = { version = "0.8", default-features = false, features = ["tls-rust # Testing proptest = "1.11" criterion = "0.8" +insta = { version = "1.40", features = ["yaml", "json"] } # Async futures = "0.3" diff --git a/TESTING.md b/TESTING.md index 38cca31..e8068b7 100644 --- a/TESTING.md +++ b/TESTING.md @@ -170,15 +170,64 @@ the upstream chain. `mix hex.audit` and `mix deps.unlock --check-unused` on every Elixir PR. +## Adopted Tier 2 standards + +### Snapshot testing — `insta` (Rust) + +Where to use it: +- Public structured outputs whose shape is part of a contract — explain + plans, AST dumps, drift report JSON, normaliser plans. + +Patterns adopted: +- `rust-core/verisim-planner/tests/snapshot_tests.rs` snapshots the + full `ExplainOutput` for three representative `LogicalPlan` fixtures + (single-modality document, two-modality graph+vector, semantic-proof + trailing). YAML format chosen so diffs read naturally. +- Workflow when a snapshot fails: + 1. `cargo insta review` — inspect the diff interactively + 2. Accept legitimate changes or fix the regression + 3. Commit the updated `tests/snapshots/*.snap` file +- In CI snapshots are read-only (no `INSTA_UPDATE`): any drift fails + the build. + +Snapshots live under `/tests/snapshots/*.snap` and are +human-readable YAML. Review them with `git log -p` like any other +source file. + +### Mutation testing — `cargo-mutants` (Rust, opt-in) + +Configuration in `.cargo/mutants.toml`: +- `exclude_globs` skips generated protobuf, bench harnesses, fuzz + harnesses, `main.rs` binaries, and build scripts (mutating them + produces uninteresting survivors). +- `timeout_multiplier = 3.0` gives slower spatial/temporal tests room. + +Recommended invocations: +- Local development: `cargo mutants -p verisim-drift` to focus on one + crate. +- PR triage: `cargo mutants --in-diff origin/main` only mutates lines + changed against main — keeps runtime under ~5 min for typical + changesets. +- Nightly/weekly: `cargo mutants -p ` full run + for a mutation-score baseline. + +CI integration: `rust-ci.yml` has a `mutants` job gated on +`workflow_dispatch` and `schedule` events (full runs take >20 min; +not suitable for every PR). Report uploaded as +`mutants-report` artifact. + +Mutation score targets: +- Priority crates (drift, normalizer, spatial, provenance, semantic, + octad): **≥80%** on core invariants +- Best-effort crates (api, repl, graph): no enforced floor + ## Next-tier standards (not yet adopted) These are documented for future work; no current CI gate enforces them. -- **Mutation testing**: `cargo-mutants --in-diff` for changed files in - `verisim-drift`, `verisim-normalizer`, `verisim-octad`. Target 75-85% - mutation score on core invariants. -- **Snapshot testing**: `insta` (Rust) for VQL parser AST, explain plans, - drift reports; `mneme` (Elixir) for built-in parser output. +- **Snapshot testing (Elixir)**: `mneme` for built-in VQL parser output. + Skipped initially because `mneme`'s interactive accept/reject doesn't + fit batch CI; we'd need to wire `MNEME_REPLY=accept` workflow. - **Differential testing**: SQLancer-style queries through redb vs Oxigraph backends and naive linear vs HNSW vector search; assert result-set equivalence to catch optimiser bugs. diff --git a/rust-core/verisim-planner/Cargo.toml b/rust-core/verisim-planner/Cargo.toml index 73ae65c..4a46f3d 100644 --- a/rust-core/verisim-planner/Cargo.toml +++ b/rust-core/verisim-planner/Cargo.toml @@ -21,3 +21,4 @@ tracing.workspace = true [dev-dependencies] proptest.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } +insta.workspace = true diff --git a/rust-core/verisim-planner/tests/snapshot_tests.rs b/rust-core/verisim-planner/tests/snapshot_tests.rs new file mode 100644 index 0000000..b99207f --- /dev/null +++ b/rust-core/verisim-planner/tests/snapshot_tests.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Snapshot tests for verisim-planner. +//! +//! The planner's ExplainOutput shape (steps, costs, hints, strategy, +//! text rendering) is part of its public contract — any change is +//! visible to users via EXPLAIN. Snapshot tests detect silent shape +//! drift: a regression in step ordering, cost calculation, or hint +//! generation will fail loudly with a clear diff. +//! +//! The cost numbers themselves are deterministic given a fixed +//! PlannerConfig, so we can snapshot them directly without redaction. +//! If non-determinism creeps in (timestamps, RNG, etc.) add +//! `[redactions]` filters to settings. +//! +//! Workflow when snapshots fail: +//! 1. Inspect the diff with `cargo insta review` +//! 2. Accept legitimate changes or fix the regression +//! 3. Commit the updated `tests/snapshots/*.snap` file +//! +//! In CI, snapshots are read-only: any drift fails the build. + +use insta::{assert_yaml_snapshot, with_settings}; +use verisim_planner::config::PlannerConfig; +use verisim_planner::optimizer::Planner; +use verisim_planner::plan::{ConditionKind, LogicalPlan, PlanNode, QuerySource}; +use verisim_planner::Modality; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +fn document_search_plan() -> LogicalPlan { + LogicalPlan { + source: QuerySource::Octad, + nodes: vec![PlanNode { + modality: Modality::Document, + conditions: vec![ConditionKind::Fulltext { + query: "machine learning".to_string(), + }], + projections: vec![], + early_limit: Some(10), + }], + post_processing: vec![], + } +} + +fn graph_then_vector_plan() -> LogicalPlan { + LogicalPlan { + source: QuerySource::Octad, + nodes: vec![ + PlanNode { + modality: Modality::Graph, + conditions: vec![ConditionKind::Traversal { + predicate: "relates_to".to_string(), + depth: Some(2), + }], + projections: vec![], + early_limit: None, + }, + PlanNode { + modality: Modality::Vector, + conditions: vec![ConditionKind::Similarity { k: 10 }], + projections: vec![], + early_limit: None, + }, + ], + post_processing: vec![], + } +} + +fn semantic_proof_plan() -> LogicalPlan { + LogicalPlan { + source: QuerySource::Octad, + nodes: vec![ + PlanNode { + modality: Modality::Document, + conditions: vec![ConditionKind::Fulltext { + query: "audit".to_string(), + }], + projections: vec![], + early_limit: None, + }, + PlanNode { + modality: Modality::Semantic, + conditions: vec![ConditionKind::ProofVerification { + contract: "integrity-v1".to_string(), + }], + projections: vec![], + early_limit: None, + }, + ], + post_processing: vec![], + } +} + +// --------------------------------------------------------------------------- +// Snapshot tests +// --------------------------------------------------------------------------- + +#[test] +fn snapshot_document_search_plan() { + let planner = Planner::new(PlannerConfig::default()); + let explain = planner.explain(&document_search_plan()).unwrap(); + + with_settings!({ description => "Single-modality document search (LIMIT 10)" }, { + assert_yaml_snapshot!("document_search_plan", explain); + }); +} + +#[test] +fn snapshot_graph_then_vector_plan() { + let planner = Planner::new(PlannerConfig::default()); + let explain = planner.explain(&graph_then_vector_plan()).unwrap(); + + with_settings!({ description => "Two-modality graph + vector; expect parallel strategy with vector first" }, { + assert_yaml_snapshot!("graph_then_vector_plan", explain); + }); +} + +#[test] +fn snapshot_semantic_proof_plan() { + let planner = Planner::new(PlannerConfig::default()); + let explain = planner.explain(&semantic_proof_plan()).unwrap(); + + with_settings!({ description => "Document + semantic proof; semantic always last" }, { + assert_yaml_snapshot!("semantic_proof_plan", explain); + }); +} + +#[test] +fn snapshot_total_cost_is_deterministic() { + // Same plan through the same config must always yield the same cost. + // This is the "no hidden state" property — if it ever fails, the + // planner has acquired a non-deterministic dependency. + let planner = Planner::new(PlannerConfig::default()); + let a = planner.explain(&graph_then_vector_plan()).unwrap(); + let b = planner.explain(&graph_then_vector_plan()).unwrap(); + + assert_eq!(a.total_cost_ms, b.total_cost_ms); + assert_eq!(a.strategy, b.strategy); + assert_eq!(a.steps.len(), b.steps.len()); + for (s1, s2) in a.steps.iter().zip(b.steps.iter()) { + assert_eq!(s1.step, s2.step); + assert_eq!(s1.operation, s2.operation); + assert_eq!(s1.estimated_cost_ms, s2.estimated_cost_ms); + } +} diff --git a/rust-core/verisim-planner/tests/snapshots/snapshot_tests__document_search_plan.snap b/rust-core/verisim-planner/tests/snapshots/snapshot_tests__document_search_plan.snap new file mode 100644 index 0000000..69fe64c --- /dev/null +++ b/rust-core/verisim-planner/tests/snapshots/snapshot_tests__document_search_plan.snap @@ -0,0 +1,21 @@ +--- +source: rust-core/verisim-planner/tests/snapshot_tests.rs +description: Single-modality document search (LIMIT 10) +expression: explain +--- +steps: + - step: 1 + operation: Document fulltext search (1 conditions) + modality: document + estimated_cost_ms: 24.24 + estimated_selectivity: 0.0005 + estimated_rows: 1 + optimization_hint: "Tantivy fulltext: \"machine learning\"" +cost_breakdown: + - modality: document + time_ms: 24.24 + percentage: 100 +performance_hints: [] +total_cost_ms: 24.24 +strategy: Sequential +text_output: "=== VeriSimDB Query Plan ===\n\nStrategy: Sequential\nTotal Estimated Cost: 24.2ms\n\n--- Steps ---\n Step 1: Document fulltext search (1 conditions) [document]\n Cost: 24.2ms | Selectivity: 0.05% | Rows: ~1\n Hint: Tantivy fulltext: \"machine learning\"\n\n--- Cost Breakdown ---\n document: 24.2ms (100.0%)\n" diff --git a/rust-core/verisim-planner/tests/snapshots/snapshot_tests__graph_then_vector_plan.snap b/rust-core/verisim-planner/tests/snapshots/snapshot_tests__graph_then_vector_plan.snap new file mode 100644 index 0000000..61188fb --- /dev/null +++ b/rust-core/verisim-planner/tests/snapshots/snapshot_tests__graph_then_vector_plan.snap @@ -0,0 +1,33 @@ +--- +source: rust-core/verisim-planner/tests/snapshot_tests.rs +description: Two-modality graph + vector; expect parallel strategy with vector first +expression: explain +--- +steps: + - step: 1 + operation: Vector similarity search (1 conditions) + modality: vector + estimated_cost_ms: 40 + estimated_selectivity: 0.001 + estimated_rows: 1 + optimization_hint: HNSW ANN search (k=10) + - step: 2 + operation: Graph traversal (1 conditions) + modality: graph + estimated_cost_ms: 225 + estimated_selectivity: 0.4 + estimated_rows: 1 + optimization_hint: "Graph traversal: relates_to (depth=2)" +cost_breakdown: + - modality: graph + time_ms: 225 + percentage: 84.90566037735849 + - modality: vector + time_ms: 40 + percentage: 15.09433962264151 +performance_hints: + - severity: info + message: "Step 2 (graph) costs 225ms — Graph traversal: relates_to (depth=2)" +total_cost_ms: 225 +strategy: Parallel +text_output: "=== VeriSimDB Query Plan ===\n\nStrategy: Parallel\nTotal Estimated Cost: 225.0ms\n\n--- Steps ---\n Step 1: Vector similarity search (1 conditions) [vector]\n Cost: 40.0ms | Selectivity: 0.10% | Rows: ~1\n Hint: HNSW ANN search (k=10)\n Step 2: Graph traversal (1 conditions) [graph]\n Cost: 225.0ms | Selectivity: 40.00% | Rows: ~1\n Hint: Graph traversal: relates_to (depth=2)\n\n--- Cost Breakdown ---\n graph: 225.0ms (84.9%)\n vector: 40.0ms (15.1%)\n\n--- Performance Hints ---\n [info] Step 2 (graph) costs 225ms — Graph traversal: relates_to (depth=2)\n" diff --git a/rust-core/verisim-planner/tests/snapshots/snapshot_tests__semantic_proof_plan.snap b/rust-core/verisim-planner/tests/snapshots/snapshot_tests__semantic_proof_plan.snap new file mode 100644 index 0000000..f841cdf --- /dev/null +++ b/rust-core/verisim-planner/tests/snapshots/snapshot_tests__semantic_proof_plan.snap @@ -0,0 +1,35 @@ +--- +source: rust-core/verisim-planner/tests/snapshot_tests.rs +description: Document + semantic proof; semantic always last +expression: explain +--- +steps: + - step: 1 + operation: Document fulltext search (1 conditions) + modality: document + estimated_cost_ms: 48 + estimated_selectivity: 0.05 + estimated_rows: 1 + optimization_hint: "Tantivy fulltext: \"audit\"" + - step: 2 + operation: Semantic verification (1 conditions) + modality: semantic + estimated_cost_ms: 510 + estimated_selectivity: 1 + estimated_rows: 1 + optimization_hint: "ZKP verify: integrity-v1" +cost_breakdown: + - modality: semantic + time_ms: 510 + percentage: 91.39784946236558 + - modality: document + time_ms: 48 + percentage: 8.60215053763441 +performance_hints: + - severity: warning + message: Total estimated cost is 510ms — consider adding LIMIT clause + - severity: info + message: "Step 2 (semantic) costs 510ms — ZKP verify: integrity-v1" +total_cost_ms: 510 +strategy: Parallel +text_output: "=== VeriSimDB Query Plan ===\n\nStrategy: Parallel\nTotal Estimated Cost: 510.0ms\n\n--- Steps ---\n Step 1: Document fulltext search (1 conditions) [document]\n Cost: 48.0ms | Selectivity: 5.00% | Rows: ~1\n Hint: Tantivy fulltext: \"audit\"\n Step 2: Semantic verification (1 conditions) [semantic]\n Cost: 510.0ms | Selectivity: 100.00% | Rows: ~1\n Hint: ZKP verify: integrity-v1\n\n--- Cost Breakdown ---\n semantic: 510.0ms (91.4%)\n document: 48.0ms (8.6%)\n\n--- Performance Hints ---\n [warning] Total estimated cost is 510ms — consider adding LIMIT clause\n [info] Step 2 (semantic) costs 510ms — ZKP verify: integrity-v1\n"