diff --git a/crates/omnigraph-cli/tests/crossversion_upgrade.rs b/crates/omnigraph-cli/tests/crossversion_upgrade.rs new file mode 100644 index 00000000..5d0d72f3 --- /dev/null +++ b/crates/omnigraph-cli/tests/crossversion_upgrade.rs @@ -0,0 +1,155 @@ +//! Cross-version upgrade: prove the CURRENT binary handles a GENUINE old-format +//! (internal schema v3) graph minted by omnigraph 0.7.2 — not a v4-shaped graph +//! with a rewound stamp. Two things the stamp-rewind stand-in +//! (`sub_current_graph_is_refused_then_rebuilt_via_export_import`) cannot prove: +//! +//! 1. the open-refusal fires on the REAL on-disk v3 shape (lineage in +//! `_graph_commits.lance`, lineage-free `__manifest`) and NAMES the writing +//! release, and +//! 2. the documented `export → init → load` rebuild round-trips the data, +//! including a `Vector` column, off a genuine v3 export. +//! +//! Gated: requires `OMNIGRAPH_OLD_BIN` (an absolute path to a 0.7.2 `omnigraph` +//! binary). Skips gracefully when unset — the same convention as the S3 and +//! system-e2e gates — so the default `cargo test --workspace` stays green +//! without it. CI sets it in the `crossversion_upgrade` job (see +//! `docs/dev/testing.md`). + +mod support; + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use support::{HERMETIC_OPERATOR_HOME, cli, fixture, output_failure, output_success}; +use tempfile::tempdir; + +/// Resolve the old (0.7.2) binary. `None` ONLY when `OMNIGRAPH_OLD_BIN` is +/// unset — the legitimate skip. A var that is SET but points at a missing path +/// is a misconfiguration (wrong install path / renamed binary) and must fail +/// loudly, never skip vacuously: in CI the var is deliberately set so the test +/// is expected to run. +fn old_bin() -> Option { + let path = PathBuf::from(std::env::var_os("OMNIGRAPH_OLD_BIN")?); + assert!( + path.exists(), + "OMNIGRAPH_OLD_BIN is set but does not exist: {} \ + (unset it to skip, or point it at a real 0.7.2 omnigraph binary)", + path.display(), + ); + Some(path) +} + +/// Run the OLD (0.7.2) binary hermetically (no developer `~/.omnigraph`). +fn run_old(bin: &Path, args: &[&str]) -> std::process::Output { + Command::new(bin) + .env("OMNIGRAPH_HOME", HERMETIC_OPERATOR_HOME) + .env_remove("OMNIGRAPH_CONFIG") + .args(args) + .output() + .expect("spawn old omnigraph binary") +} + +fn assert_ok(label: &str, out: &std::process::Output) { + assert!( + out.status.success(), + "old binary `{label}` failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); +} + +fn nonblank_lines(bytes: &[u8]) -> usize { + String::from_utf8_lossy(bytes) + .lines() + .filter(|l| !l.trim().is_empty()) + .count() +} + +#[test] +fn current_binary_refuses_and_rebuilds_a_genuine_v3_graph() { + let Some(old) = old_bin() else { + eprintln!( + "skipping cross-version upgrade test: OMNIGRAPH_OLD_BIN is not set to a 0.7.2 binary" + ); + return; + }; + + let temp = tempdir().unwrap(); + let old_graph = temp.path().join("old-v3.omni"); + // `search.pg` / `search.jsonl` are byte-identical in v0.7.2 and exercise a + // `Vector(4)` column plus indexed strings — a fixture both binaries parse. + let schema = fixture("search.pg"); + let data = fixture("search.jsonl"); + let og = old_graph.to_str().unwrap(); + + // 1. Mint a GENUINE v3 graph with the old binary. + assert_ok( + "init", + &run_old(&old, &["init", "--schema", schema.to_str().unwrap(), og]), + ); + assert_ok( + "load", + &run_old( + &old, + &["load", "--mode", "overwrite", "--data", data.to_str().unwrap(), og], + ), + ); + + // Prove it is really v3 on disk: pre-v4 graphs carry the now-retired + // `_graph_commits.lance` lineage dataset (a v4 graph has neither). + assert!( + old_graph.join("_graph_commits.lance").exists(), + "a genuine v3 graph must have the legacy _graph_commits.lance dataset", + ); + + // 2. Old binary export → JSONL. + let export = run_old(&old, &["export", og]); + assert_ok("export", &export); + assert!(!export.stdout.is_empty(), "old export produced no rows"); + let v3_jsonl = temp.path().join("v3.jsonl"); + std::fs::write(&v3_jsonl, &export.stdout).unwrap(); + + // 3. The CURRENT binary refuses the genuine v3 graph, names the writing + // release, and nudges to export — on the real on-disk shape. + let refusal = output_failure(cli().arg("snapshot").arg(&old_graph)); + let stderr = String::from_utf8_lossy(&refusal.stderr); + assert!( + stderr.contains("export"), + "refusal must nudge the operator to export, got: {stderr}", + ); + assert!( + stderr.contains("0.6.2 to 0.7.2"), + "refusal must name the full release range that wrote this stamp (v3 → 0.6.2 to 0.7.2), \ + got: {stderr}", + ); + + // 4. The CURRENT binary rebuilds: fresh init + load the v3 export. + let new_graph = temp.path().join("new-v4.omni"); + output_success(cli().arg("init").arg("--schema").arg(&schema).arg(&new_graph)); + output_success( + cli() + .arg("load") + .arg("--mode") + .arg("overwrite") + .arg("--data") + .arg(&v3_jsonl) + .arg(&new_graph), + ); + + // 5. Round-trip fidelity: re-export with the current binary and compare. + let reexport = output_success(cli().arg("export").arg(&new_graph)); + assert_eq!( + nonblank_lines(&export.stdout), + nonblank_lines(&reexport.stdout), + "row count must round-trip v3 → v4", + ); + let rebuilt = String::from_utf8_lossy(&reexport.stdout); + assert!( + rebuilt.contains("embedding"), + "the Vector column must survive the rebuild, got: {rebuilt}", + ); + assert!( + rebuilt.contains("ml-intro"), + "the rebuilt graph must preserve node data, got: {rebuilt}", + ); +} diff --git a/crates/omnigraph/src/db/manifest/migrations.rs b/crates/omnigraph/src/db/manifest/migrations.rs index 6b79ad6b..cf3b2f33 100644 --- a/crates/omnigraph/src/db/manifest/migrations.rs +++ b/crates/omnigraph/src/db/manifest/migrations.rs @@ -70,6 +70,26 @@ pub(crate) const INTERNAL_MANIFEST_SCHEMA_VERSION: u32 = 4; /// module doc). pub(crate) const MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION: u32 = INTERNAL_MANIFEST_SCHEMA_VERSION; +/// The omnigraph release line that wrote a given internal-schema stamp. The +/// open-refusal uses it to tell an operator exactly which binary to use to +/// export a sub-CURRENT graph (the export side of the strand-model upgrade — +/// see `docs/user/operations/upgrade.md`). Ranges are the release tags that +/// stamped each version (verify with +/// `git show vX.Y.Z:crates/omnigraph/src/db/manifest/migrations.rs`): +/// v1 ≤ 0.3.1, v2 0.4.1–0.6.1, v3 0.6.2–0.7.2, v4 0.8.x. +pub(crate) fn release_for_internal_schema_version(stamp: u32) -> &'static str { + match stamp { + 1 => "0.3.1 or earlier", + 2 => "0.4.1 to 0.6.1", + 3 => "0.6.2 to 0.7.2", + 4 => "0.8.x", + // Unreachable today (1–3 are mapped; ≥ CURRENT is caught by the ceiling + // guard before this is consulted). Worded to read naturally after + // "created by omnigraph " if a future bump ever leaves a gap. + _ => "an unrecognized older release", + } +} + const INTERNAL_SCHEMA_VERSION_KEY: &str = "omnigraph:internal_schema_version"; /// Read the on-disk stamp from `__manifest`'s schema-level metadata. @@ -109,10 +129,14 @@ pub(crate) fn refuse_if_stamp_unsupported(stamp: u32) -> Result<()> { if stamp < MIN_SUPPORTED_INTERNAL_SCHEMA_VERSION { return Err(OmniError::manifest(format!( "__manifest is stamped at internal schema v{stamp}, but this omnigraph reads only v{current}. \ - This graph was created by an older omnigraph release; rebuild it: run `omnigraph export` with \ - the older omnigraph binary that created it, then `omnigraph init` + `omnigraph load` with this one. \ - (Data, vectors, and blobs are preserved; commit history and branches are not.)", + This graph was created by omnigraph {release}. Rebuild it: with an omnigraph {release} binary run \ + `omnigraph export > graph.jsonl`, then with this binary run \ + `omnigraph init --schema ` and \ + `omnigraph load --mode overwrite --data graph.jsonl `. \ + (Data, vectors, and blobs are preserved; commit history and branches are not.) \ + See docs/user/operations/upgrade.md.", current = INTERNAL_MANIFEST_SCHEMA_VERSION, + release = release_for_internal_schema_version(stamp), ))); } Ok(()) @@ -160,4 +184,21 @@ mod tests { "a future stamp must be refused" ); } + + /// The refusal names the release line that wrote each stamp so an operator + /// knows which binary to use for the export step; unknown stamps fall back + /// without panicking. + #[test] + fn release_names_the_writing_line_for_each_stamp() { + assert_eq!(release_for_internal_schema_version(3), "0.6.2 to 0.7.2"); + assert_eq!(release_for_internal_schema_version(4), "0.8.x"); + assert_eq!( + release_for_internal_schema_version(99), + "an unrecognized older release" + ); + // The sub-CURRENT refusal embeds the named release. + let err = refuse_if_stamp_unsupported(3).unwrap_err().to_string(); + assert!(err.contains("0.6.2 to 0.7.2"), "got: {err}"); + assert!(err.contains("omnigraph export"), "got: {err}"); + } } diff --git a/crates/omnigraph/src/db/manifest/tests.rs b/crates/omnigraph/src/db/manifest/tests.rs index 435f8d5a..1d6e4b33 100644 --- a/crates/omnigraph/src/db/manifest/tests.rs +++ b/crates/omnigraph/src/db/manifest/tests.rs @@ -1826,10 +1826,16 @@ async fn sub_current_graph_is_refused_then_rebuilt_via_export_import() { Ok(_) => panic!("a sub-CURRENT graph must be refused on open"), Err(err) => err, }; + let msg = err.to_string(); assert!( - err.to_string().contains("export"), + msg.contains("export"), "the refusal must nudge the operator to `omnigraph export`, got: {err}", ); + assert!( + msg.contains("0.7.2"), + "the refusal must name the release that wrote this stamp (v3 → 0.6.2 to 0.7.2) so the \ + operator knows which binary to use, got: {err}", + ); // Rebuild with this binary: fresh init + load the export. let dir_new = tempfile::tempdir().unwrap(); diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 9138a105..b038e9fd 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -7,7 +7,7 @@ This file is the always-on map of the test surface. **Consult it before every ta | Crate | Path | Style | |---|---|---| | `omnigraph` (engine) | `crates/omnigraph/tests/` | Integration tests (36 files), fixture-driven, share `tests/helpers/mod.rs` | -| `omnigraph-cli` | `crates/omnigraph-cli/tests/` | Per-area suites (post-modularization): `cli_cluster.rs` (cluster command surface + operator-actor cascade), `cli_cluster_e2e.rs` (spawned-binary lifecycle compositions — lost-state re-import recovery, out-of-band drift, graph-root destruction, multi-graph mixed-disposition convergence), `cli_data.rs` (load/read/change/branch/commit/export/snapshot/policy/embed/maintenance + operator format cascade), `cli_schema_config.rs` (init/config, schema plan/apply), `cli_queries.rs`, `parity_matrix.rs` (RFC-009 Phase 1: the embedded-vs-remote referee — every forked verb run against both arms with matched Cedar policy and the same actor, scrubbed-JSON + exit-code equality; divergences are pinned in its `KNOWN_DIVERGENCES` ledger, never silently repaired), `system_local.rs` (full-cycle cluster lifecycle with a spawned `--cluster` server, applied-policy enforcement over HTTP, keyed-credential auth, operator aliases), `system_remote.rs`; share `tests/support/mod.rs` (hermetic `OMNIGRAPH_HOME` by default) | +| `omnigraph-cli` | `crates/omnigraph-cli/tests/` | Per-area suites (post-modularization): `cli_cluster.rs` (cluster command surface + operator-actor cascade), `cli_cluster_e2e.rs` (spawned-binary lifecycle compositions — lost-state re-import recovery, out-of-band drift, graph-root destruction, multi-graph mixed-disposition convergence), `cli_data.rs` (load/read/change/branch/commit/export/snapshot/policy/embed/maintenance + operator format cascade), `cli_schema_config.rs` (init/config, schema plan/apply), `cli_queries.rs`, `parity_matrix.rs` (RFC-009 Phase 1: the embedded-vs-remote referee — every forked verb run against both arms with matched Cedar policy and the same actor, scrubbed-JSON + exit-code equality; divergences are pinned in its `KNOWN_DIVERGENCES` ledger, never silently repaired), `system_local.rs` (full-cycle cluster lifecycle with a spawned `--cluster` server, applied-policy enforcement over HTTP, keyed-credential auth, operator aliases), `system_remote.rs`, `crossversion_upgrade.rs` (gated genuine-v3 upgrade round-trip — see "Cross-version upgrade" below); share `tests/support/mod.rs` (hermetic `OMNIGRAPH_HOME` by default) | | `omnigraph-cluster` | mostly in-source `#[cfg(test)] mod tests`; `tests/failpoints.rs` (feature-gated); `tests/s3_cluster.rs` (bucket-gated full lifecycle on object storage) | Cluster config parser, local JSON state diff, state CAS/lock handling/recovery, read-only validate/plan/status plus explicit refresh/import graph observations, config-only apply (content-addressed payload publish, disposition gating, composite-digest convergence, idempotent re-apply), catalog payload verification (status read-only, refresh drift + self-heal), failpoint crash-mid-apply / CAS-race coverage, Stage 4A graph creation (create executor, recovery sidecars + sweep rows, create crash windows), Stage 4B schema apply (migration previews in plan, schema executor, schema-apply sweep classification, schema crash windows), Stage 4C gated deletes (digest-bound approvals, delete executor + tombstones, delete sweep rows, delete crash windows), and 5A policy binding metadata (applies_to in the applied revision, binding-change diffing + convergence, pre-5A backfill), and the 5B serving-snapshot read API (converged read, refusal rows) | | `omnigraph-server` | `crates/omnigraph-server/tests/` | Per-area suites (post-modularization): `auth_policy.rs`, `data_routes.rs`, `schema_routes.rs`, `stored_queries.rs`, `multi_graph.rs` (cluster-mode boot — converged serving, policy binding wiring, boot refusals — + the concurrent branch-ops matrix), `boot_settings.rs` (mode inference, PolicySource), `s3.rs` (bucket-gated: single-graph serving + config-free `--cluster s3://` boot), `openapi.rs` (OpenAPI drift / regeneration); share `tests/support/mod.rs` | | `omnigraph-compiler` | mostly in-source `#[cfg(test)] mod tests` | Parser, type-checker, IR lowering, lint | @@ -90,6 +90,10 @@ CI runs these S3-backed **correctness** tests against a containerized RustFS ser Locally, set `OMNIGRAPH_S3_TEST_BUCKET` (and the usual `AWS_*` vars including `AWS_ENDPOINT_URL_S3` for non-AWS) before running. Without those, S3 tests skip gracefully. +## Cross-version upgrade (genuine v3 → current) + +`crates/omnigraph-cli/tests/crossversion_upgrade.rs` is the empirical proof that the strict-single-version refusal works on a **real** old-format graph — not the stamp-rewind stand-in in `db/manifest/tests.rs::sub_current_graph_is_refused_then_rebuilt_via_export_import` (which only rewinds the stamp number on a v4-shaped graph). It mints a genuine internal-schema-v3 graph with an omnigraph **0.7.2** binary (real `_graph_commits.lance`, lineage-free `__manifest`), then asserts the current binary refuses it with the release-named message and that the documented `export → init → load` rebuild round-trips (row count + the `Vector` column). It is **gated on `OMNIGRAPH_OLD_BIN`** (an absolute path to a 0.7.2 `omnigraph`) and skips gracefully when unset, so the default `cargo test --workspace` stays green without it. **Deliberately NOT in CI** (same disposition as `write_cost_s3`): compiling 0.7.2 from source is a 15–25 min job, and the format-stamp seam changes a few times a year — run it on demand when touching `migrations.rs`, the loader/export surfaces, or at a release boundary. To run: `cargo install omnigraph-cli@0.7.2 --locked --root /tmp/og-old` then `OMNIGRAPH_OLD_BIN=/tmp/og-old/bin/omnigraph cargo test -p omnigraph-cli --test crossversion_upgrade`. + ## System e2e requirements and suppression The CLI system tests (`system_local.rs`) spawn the workspace-built `omnigraph` and `omnigraph-server` binaries (cargo provides paths via `CARGO_BIN_EXE_*`), bind ephemeral localhost ports, and use local-FS temp dirs — no external services, no env vars required; they run in the default `cargo test --workspace`. The comprehensive cluster lifecycle e2es (multi-server-restart flows) honor an opt-out for constrained sandboxes: set `OMNIGRAPH_SKIP_SYSTEM_E2E=1` to skip them with a logged message (the same graceful-skip pattern as the S3 gate). Cargo-native filtering also works: `cargo test --test system_local -- --skip local_cluster`. diff --git a/docs/dev/versioning.md b/docs/dev/versioning.md index 7c46c3ad..f06a8b16 100644 --- a/docs/dev/versioning.md +++ b/docs/dev/versioning.md @@ -27,6 +27,11 @@ it cannot serve: - a stamp **above** CURRENT → refused with "upgrade omnigraph", so an old binary cannot silently misread a newer format. +The below-CURRENT refusal names the release line that wrote the stamp +(`release_for_internal_schema_version` in `db/manifest/migrations.rs`) and prints +the exact `export` / `init` / `load` commands, so the upgrade is fail-closed **and** +self-service — the operator can fetch the right old binary without guessing. + There is no in-place migration dispatcher. The single source file `db/manifest/migrations.rs` holds only the version constant, the stamp read/write, and `refuse_if_stamp_unsupported`. diff --git a/docs/rfcs/rfc-0015-ingest-embeddings.md b/docs/rfcs/rfc-0015-ingest-embeddings.md new file mode 100644 index 00000000..fe139b11 --- /dev/null +++ b/docs/rfcs/rfc-0015-ingest-embeddings.md @@ -0,0 +1,258 @@ +# RFC-015: Ingest-time `@embed` via the embedding reconciler + +**Status:** Proposed +**Date:** 2026-06-21 +**Tickets:** `iss-ingest-embed` (umbrella); relates to RFC-012 (embedding client), +invariant 7 (indexes are derived state) +**Author track:** maintainer-internal RFC + +> Evidence tags: **[S]** verified in v0.7.1 source (`file:line`), **[U]** verified +> against an external system, **[D]** documented behavior. Unmarked = design intent. + +## Summary + +`@embed("source_text", model="…")` records *which* String property seeds a +Vector column and with *which* model, but **does not embed at write time** — the +loader and mutation paths never call the embedder, and stored vectors must be +supplied in the load data or pre-filled by the offline `omnigraph embed` file +pipeline ([embeddings.md:92](../user/search/embeddings.md) [D]). + +This RFC closes that gap **without putting a network call on the write path.** +An embedding is *derived physical state* — the same class as an index — so it is +materialized by a **reconciler**, not inline: `load`/`mutate` store the source +text and leave the dependent vector **pending** (NULL); an `ensure_embeddings` +reconciler (sibling of today's `ensure_indices`) embeds the pending rows in +batches from manifest state, with bounded retry. A not-yet-embedded row is +queryable by traversal/filter immediately and becomes a `nearest()` hit once +materialized — documented partial coverage, exactly as for a deferred index. + +This is invariant 7 applied to embeddings, and it is the design the Postgres +ecosystem converged on (Timescale pgai Vectorizer, Supabase pgmq) [U]. + +## Motivation + +The natural request — "embed automatically when I add data" — has one obvious +implementation (embed inline during `mutate`/`load`) that is **wrong for this +codebase on three independent counts**: + +1. **It is on the deny-list.** [invariants.md](invariants.md) forbids + "synchronous inline vector/FTS index rebuilds on the write path." An + embedding is the same shape (expensive, network-bound, derivable). The + deny-list does not distinguish "build the index" from "compute the value the + index covers." +2. **It blows the write-path cost contract.** RFC-013 (and step 3b, + [rfc-013-step-3b-writetxn.md](rfc-013-step-3b-writetxn.md)) is making the + write path O(1) and failure-bounded. A per-row embedding call is + O(network-latency) and fallible — a single-row `mutate` would block on a + model round-trip (the latency that makes this unacceptable for interactive + mutations). +3. **It couples a logical write to a flaky external service.** Per pgai's stated + rule: *"primary data-modification operations (INSERT/UPDATE/DELETE) should not + be dependent on the embedding operation, otherwise the application is down + every time the endpoint is slow or fails"* [U]. That is invariant 7's "a + physical operation must never fail a logical one," verbatim from the field. + +### Prior art (the three patterns) + +| Pattern | Who | Fit | +|---|---|---| +| **Pre-embed / BYO vector** | raw pgvector, Qdrant, Milvus (classic), Atlas, Cloudflare Vectorize | omnigraph already has this — the `omnigraph embed` *file* pipeline. Keep it. | +| **Synchronous-inline at write** | **HelixDB** (`AddV(Embed(content), …)`) [U], Weaviate vectorizers, Milvus 2.5 functions, Pinecone integrated inference, LanceDB embedding registry | Blocks the write; fine for bulk-batched, bad for single-row mutations. **HelixDB — the closest analog — took this path and inherited the latency.** Forbidden here by the deny-list. | +| **Async materialization / reconciler** | **Timescale pgai Vectorizer**, Supabase (trigger → `pgmq` → worker) [U] | The write enqueues; a batched, retrying worker embeds and writes back; "a delay may occur before [vectors] become available." **= invariant 7. Recommended.** | + +"Lazy on read" is not a fourth option: the ANN index needs the vector *present* +to index it, so you cannot defer the vector itself to query time without giving +up the index. The reconciler is "lazy relative to the write, eager relative to +search" — which is precisely invariant 7's partial-coverage licence. + +## Design + +### Embeddings are derived state, keyed off `@embed` + +`@embed("source", model)` declares a derivation: `Vector_col = embed(source_col)` +[D]. That makes the vector column *physical, rebuildable, source-derived* state — +identical in kind to an index. The whole design follows from treating it that way. + +### Write path: store source, leave the vector pending (NULL) + +`load`/`mutate` are **unchanged in cost**: they write the source text and any +caller-supplied vector. They do **not** call the embedder. A row whose `@embed` +source is present but whose vector is NULL is *pending*. The write path's only +new behavior is **staleness invalidation** (see §"Staleness"): when a write +touches an `@embed` *source* column, it clears (NULLs) the dependent vector so +the reconciler re-embeds. Clearing a column is write-local and cheap (no +network). + +This reuses the exact shape already in `table_ops.rs`: an untrainable Vector +column (all-NULL today) is isolated as a `PendingIndex` rather than aborting the +build [S]; here the *rows* are pending, tracked the same way. + +### The reconciler: `ensure_embeddings` + +A sibling of `ensure_indices` +(`db/omnigraph/table_ops.rs`, materialized in `optimize.rs` [S]): + +1. For each `@embed`-annotated `(table, vector_col, source_col, model)`, scan for + **pending rows**: `vector_col IS NULL AND source_col IS NOT NULL`. Cost is + bounded by *pending rows*, not history (invariant 15). +2. Embed the source texts **in batches** via a new + `EmbeddingClient::embed_documents(&[&str]) -> Vec>` (see §"Required + new mechanics"), honoring the existing retry/backoff + (`OMNIGRAPH_EMBED_RETRY_*`, `embedding.rs:18` [S]) and the recorded model. +3. Write the vectors back via the normal staged-write + manifest publish path + (an `update` keyed on row id). Then fold the now-trainable vector column into + its index via the existing `ensure_indices` pass — embeddings unblock the + deferred vector index automatically. +4. A model mismatch (recorded model ≠ resolved client model) **refuses loudly**, + reusing the query-path same-space validation that already exists [D]. + +**Trigger (decision §A):** start **explicit** — `ensure_embeddings` runs inside +`optimize` and as a new `omnigraph embed --graph ` mode (the file +pipeline gains a graph target). This matches how indexes already reconcile +(operator/cron-triggered, no background loop). A **background loop** (pgai's +every-5-min worker [U]) is the eventual convergence, specified as a follow-up, +not built first. + +### Staleness (decision §C) + +A source-text `update` must re-embed. Two options: + +- **Recommended — NULL-on-source-change.** When a write touches an `@embed` + source column, clear the dependent vector. Then *one* universal pending + predicate (`vector IS NULL AND source IS NOT NULL`) covers both first-insert + and post-update, with no per-row bookkeeping. Cost: the write path must know + the `@embed` dependency (a catalog lookup it already has) and clear a column. +- *Alternative — embedded-from marker.* Store the source version/hash the vector + was derived from; the reconciler re-embeds rows whose + `_row_last_updated_at_version` (`lance_version_columns` [S]) exceeds it. No + write-path coupling, but adds a per-row marker column. + +### Read semantics (decision §B) + +A pending row is **present and queryable** by traversal/filter/scalar predicates +immediately; it is simply **not yet a `nearest()` hit.** This is invariant 7 +partial coverage, surfaced — not silent: + +- Expose a **pending-embedding count** (per table/column) via `status`/`optimize` + output, so operators see the backlog. +- Provide an **explicit synchronous opt-in** for the rare read-after-write-on- + vectors case — e.g. `omnigraph embed --graph --wait` or a `mutate` flag that + runs `ensure_embeddings` for the touched rows before returning (opt-in only; + never the default, never on the hot path). + +## Required new mechanics + +| Mechanic | Today | Needed | +|---|---|---| +| **Batch embed** | `embed_document_text(&str)` / `embed_query_text(&str)` are one-string-per-call (`embedding.rs:248,252` [S]) | Add `embed_documents(&[&str]) -> Result>>` (provider batch endpoint where available; chunked sequential fallback). pgai treats batch as load-bearing for throughput/rate-limits [U]. | +| **Pending-row scan** | `PendingIndex` tracks declared-but-unbuilt *indexes* (`table_ops.rs` [S]) | A row-level pending scan (`vector IS NULL AND source IS NOT NULL`) per `@embed` column. | +| **`@embed` dependency in the write path** | `@embed` is a catalog annotation consumed at query typecheck/lint [D] | The loader/mutation must read it to NULL-on-source-change (decision §C). | +| **Graph-target embed** | `omnigraph embed` is file→file [D] | `omnigraph embed --graph ` runs the reconciler against a live graph. | + +The provider client, model-identity recording, same-space validation, and +retry/backoff already exist (RFC-012) — this is write-path + reconciler +integration, not a new subsystem. + +## Invariants & deny-list check + +- **7 (indexes are derived state):** this *is* an instance — embeddings converge + from manifest/source state, never extend the critical write path; reads are + correct (just not vector-ranked) under partial coverage. +- **Deny-list "synchronous inline vector/FTS index rebuilds on the write path":** + obeyed by construction — embedding is off the write path. +- **9 (integrity failures are loud):** a model mismatch or an embed failure on + the reconciler is surfaced (pending count, typed error), not a placeholder + vector. +- **13 (operational failures bounded/observable):** the reconciler's embed calls + are batched, retry-bounded, and report a pending backlog; a flaky provider + degrades vector-search freshness, never write availability. +- **2/3 (manifest-atomic, one snapshot):** unchanged — the reconciler's write-back + is an ordinary staged write + manifest publish. + +## Cost contract + +- **Write path:** unchanged — O(1), no embedder call (the NULL-on-source-change + clear is a local column write). Assert in `write_cost.rs` that a `mutate` + touching an `@embed` source performs **zero** embedding round-trips. +- **Reconciler:** O(pending rows), batched; flat in history depth (it scans by + the pending predicate, not the version chain). + +## Testing + +- **Write-path-clean:** a `mutate`/`load` into an `@embed` table issues no + embedder call and leaves the vector NULL (mock client asserts zero invocations). +- **Reconcile fills:** `ensure_embeddings` embeds the pending rows, the vector + column becomes non-NULL, and the deferred vector index then builds. +- **Staleness:** updating the source NULLs the vector; the next reconcile + re-embeds; updating a *non-source* property does **not**. +- **Read partial coverage:** a pending row is returned by a traversal/filter + query but not by `nearest()`; the pending count is reported. +- **Model mismatch refuses** on the reconciler (reuse the query-path validation). +- **Batch:** `embed_documents` over N texts maps 1:1 to N output vectors in order; + a mid-batch provider error is retry-bounded and reported, not silently dropped. +- Mock provider throughout (no network in CI), mirroring the existing embedding + tests. + +## Rollout + +1. **PR 1 — `embed_documents` batch method** + reconciler-internal use. Pure + addition to the client. +2. **PR 2 — `ensure_embeddings` reconciler** + `optimize` integration + the + pending-row scan. Read-only effect until something is pending. +3. **PR 3 — `omnigraph embed --graph`** (graph target for the existing CLI) + + the pending-count surface in `status`/`optimize`. +4. **PR 4 — write-path staleness** (NULL-on-source-change) + the + `--wait`/synchronous opt-in. This is the only write-path touch; lands last, + behind the reconciler that consumes its output. +5. *(follow-up)* background reconciler loop (pgai-style), if/when operators want + automatic freshness without a cron. + +PRs 1–3 deliver "add data, then reconcile, then it's searchable" with **no +write-path change at all**; PR 4 adds automatic staleness + the opt-in. + +## Drawbacks & alternatives + +- **Read-after-write delay on vectors.** The cost of keeping writes fast: a + freshly-mutated node is not a `nearest()` hit until reconciled. Mitigated by + the pending-count surface and the synchronous opt-in. This is the same + trade-off pgai ships and documents [U]. +- **Explicit-trigger first** means freshness depends on `optimize`/cron cadence, + not seconds. Accepted as the lower-liability start (matches index reconcile); + the background loop is the convergence. +- **Rejected — synchronous inline (HelixDB regime).** The latency and + failure-coupling this RFC exists to avoid; on the deny-list. +- **Rejected — embed-on-read (lazy).** Doesn't fit a vector column the ANN index + must contain. + +## Reversibility + +High. The vector column, `@embed` annotation, and on-disk format are unchanged; +the reconciler is additive. The one observable contract change (a pending window +before vectors are searchable) is the documented partial-coverage semantics, and +the synchronous opt-in covers callers that cannot tolerate it. + +## Open questions / decisions to ratify + +- **§A Trigger model:** explicit (`optimize` + `omnigraph embed --graph`) first, + background loop later — confirm the staged approach. +- **§B Read-after-write:** ratify the "present but not vector-searchable until + reconciled" contract + the synchronous opt-in shape. +- **§C Staleness signal:** NULL-on-source-change (recommended) vs embedded-from + marker. +- **Batching granularity / provider limits:** max batch size + handling a + provider's per-request token cap (chunk within `embed_documents`). +- **Chunking:** pgai also *chunks* long source text before embedding [U]. Out of + scope here (one source property → one vector), but note it as a future + extension if long-text columns appear. + +## Relationship to prior work + +- **RFC-012 (embedding provider config):** provides the client, model identity, + same-space validation, retry/backoff this RFC consumes. RFC-015 is the + write-path/reconciler integration RFC-012 deferred. +- **Invariant 7 / the deferred-vector-index path** (`table_ops.rs` + `PendingIndex`, `optimize.rs` [S]): the mechanism this extends from + index-coverage to row-level embedding coverage. +- **RFC-013 (write-path latency) / step 3b:** the cost-and-failure-bounding + contract that makes synchronous inline embedding unacceptable and the + reconciler necessary. diff --git a/docs/rfcs/rfc-0018-ingest-wal.md b/docs/rfcs/rfc-0018-ingest-wal.md new file mode 100644 index 00000000..665e4ca5 --- /dev/null +++ b/docs/rfcs/rfc-0018-ingest-wal.md @@ -0,0 +1,507 @@ +--- +type: spec +title: "RFC-018 — Streaming-ingest WAL on Lance MemWAL" +description: Adds a durability-first streaming ingest path (ack on WAL durability, asynchronous fold into the graph commit chain) built entirely on Lance's MemWAL primitive; reconciled against Lance v8.0.0 and the v9 beta line; analyzed for composition with the upstream multi-table-commit RFCs. +status: draft +tags: [eng, rfc, wal, ingest, lance, omnigraph] +timestamp: 2026-07-02 +owner: +--- + +# RFC-018 — Streaming-ingest WAL on Lance MemWAL + +**Status:** Draft / for discussion +**Date:** 2026-07-02 +**Surveyed version:** 0.7.2 (branch `dst-extract-crate`); Lance pinned at 7.0.0 +**Upstream surveyed:** Lance v8.0.0 (released; RC votes closed 2026-07-01), v9.0.0-beta.10; MemWAL spec (`lance.org/format/table/mem_wal/`, fetched in full 2026-07-02); discussions #7260, #7264, #7222, #7176 +**Audience:** OmniGraph maintainers + +--- + +## 0. TL;DR + +OmniGraph's write path is optimized for *interactive* graph mutation: every +`mutate_as`/`load` pays the full commit chain (staged Lance commit per touched +table + one `__manifest` publish CAS) before acknowledging, and per-branch +throughput is capped by the `graph_head` row CAS at roughly the object store's +conditional-write rate. That is the right trade for its workload and this RFC +does not touch it. What OmniGraph lacks is a **streaming ingest** path: high- +frequency small writes (event streams, agent telemetry, embedding pipelines) +where per-write ack latency matters and per-write graph commits are wasteful. + +This RFC adds one, with a hard constraint set: + +1. **The WAL sits in FRONT of the commit point, never beside it.** The + `__manifest` CAS remains the single linearization point for graph + visibility and lineage (the RFC-013 Phase 7 conclusion). The WAL is an + ingestion buffer whose contents *become* graph state only through the + existing publish seam. No second metadata authority is created. +2. **No custom WAL.** Lance ships MemWAL — a complete spec'd LSM layer + (shards, epoch-fenced writers, Arrow-IPC WAL entries, flushed memtables as + Lance tables with pre-built indexes, a MemWAL system index whose merge + progress commits atomically with the merge). Invariant 1 says use it, and + §3 inventories every piece of it we consume. +3. **Ack means durable.** A write is acknowledged only after its WAL entry is + durably on the object store (Lance durability waiters). The deny-list item + "no acks before durable persistence" holds; what moves is *graph + visibility*, which arrives at the next fold. Fresh-tier reads that see + folded-but-unpublished / unfolded WAL data are **explicit, opt-in, + read-only, and documented** (invariant 6's escape hatch, used exactly as + written). + +Target substrate is **Lance v8.0.0** (Phase 0 is the bump): v8 is the MemWAL +hardening release (writer-fencing WAL sentinel, durability waiters, LSM read +planners). Delete support arrives with **v9** (tombstones); stream-mode +deletes are phased on it. The design composes cleanly with — and is partially +scaffolding-for — the upstream multi-table commit work (#7260/#7264): §7 +analyzes both directions and pins the one design rule (single publish seam) +that makes adopting either a publisher swap rather than a redesign. + +--- + +## 1. Motivation + +Today one graph write costs, serially: per touched table a staged write + +`commit_staged` (a Lance commit, itself several object-store round trips), +then one `ManifestBatchPublisher::publish` (merge-insert CAS on `__manifest`). +Latency is the sum of the chain; throughput per branch is bounded by the +shared `graph_head:` row (the deliberate RFC-013 §7.1 contention +point) at the object store's conditional-write rate — single-digit commits/sec +on S3. `write_cost.rs` keeps this *flat in history*, which is the right +guarantee for the interactive path, but nothing can make it *cheap per op*: +the floor is object-store round trips. + +Workloads that don't fit: continuous event/entity streams from agents, +sensor-style feeds, bulk embedding backfills that trickle, any producer that +needs sub-100ms acks or >tens of writes/sec sustained on one branch. Today +such producers must client-side batch into `load` calls, re-implementing an +ingestion buffer badly, with no durability until the batch commits. + +The turbopuffer/SlateDB survey (2026-07, conversation record) located the +correct shape: **ack on WAL durability, group-commit, fold asynchronously into +the columnar substrate, keep one CAS commit point.** turbopuffer acks at WAL +durability and bridges reads with a WAL-tail scan; SlateDB acks on WAL flush +and serves cross-node readers only durable published state. Both keep exactly +one commit authority. OmniGraph's commit authority is `__manifest`; the WAL +therefore buffers *in front of* it. + +### 1.1 Relationship to PR #318 (warm publish + pinned opens) + +PR #318 and this RFC attack different terms of the same budget and compose: +#318 removes the *history-dependent* term (warm attempt-0 publish from +`known_state` drives the per-write `__manifest` scan to 0; pinned `At(v)` +staging opens drop the `_versions/` LIST), making a warm write O(1) in commit +depth. This RFC amortizes the *constant* term that remains — the commit-chain +round-trip floor and the `graph_head` CAS rate — which #318 cannot touch and +which becomes the only wall once #318 lands. Structurally the fold inherits +#318 for free: it publishes only through the `ManifestBatchPublisher` seam, +whose internals (`PublishPlan`, warm/cold attempts) are exactly what #318 +changes, and the fold's long-lived coordinator is the ideal warm-path client. +Sequencing notes: (a) §9's concurrency cells should use the +`Cohort` multi-coordinator DST harness #318 lands rather than invent +one; (b) Phase 0/1 must sequence against #318's U2 follow-up (internal schema +v4 → v5, the `table_version` row-accumulation wall) so enrollment's +unenforced-PK step lands against v5 rather than straddling the migration; +(c) the ~5% cross-process RC-1 uncovered-drift exposure #318 +characterizes-but-tolerates is one more argument for MemWAL's epoch fencing: +ingest tables become the first write surface with a true cross-process +fencing primitive. + +## 2. Non-goals and preserved invariants + +- **Not a replacement for `mutate_as`/`load`.** The per-query-atomic + interactive path is unchanged, including its "one query = one graph commit" + lineage contract. Stream ingest is a distinct, opt-in surface with a + documented coarser lineage granularity (one fold = one commit). +- **Not a metadata store.** Lineage, branch heads, table versions stay in + `__manifest` rows. Nothing in this RFC writes graph metadata anywhere else. +- **Not a custom WAL / transaction manager** (deny-list). Everything below + the OmniGraph fold logic is Lance MemWAL, spec'd upstream. +- **Not early ack.** `await_durable=false`-style acks (SlateDB's opt-in) are + rejected; invariant 6 and the deny-list stand. +- **Not cross-query transactions.** Branches remain the multi-query + transaction mechanism. + +## 3. Substrate inventory — what Lance provides and how we use all of it + +Per the lance.md protocol the MemWAL spec and adjacent pages were fetched in +full (2026-07-02). The spec is **experimental** upstream — risk register in +§10. Inventory, mapped to consumption: + +| Lance tooling | Spec/PR | How this RFC uses it | +|---|---|---| +| MemWAL shards; one epoch-fenced writer per shard | spec §Shard | One shard per enrolled `(table, branch)` in Phase 1 (OmniGraph is effectively single-writer per graph today); shard count is a later tunable | +| Sharding specs + transforms (`bucket(murmur3)`, `identity`, …) | spec §Sharding Spec | Phase 2 scale-out: `bucket(key, N)` on the table's `@key` column so each primary key maps to exactly one shard (the spec's last-write-wins correctness requirement) | +| WAL entries: Arrow IPC files, bit-reversed naming (S3 partition spread), strictly sequential positions | spec §WAL | The durable unit. Group commit = batching writes per WAL flush; flush interval/size configurable via `writer_config_defaults` | +| Writer epoch fencing + **WAL sentinel on claim** | spec §Writer Fencing; v8 #7110 | Cross-process safety for ingest writers — a fenced zombie cannot ack lost writes. Note #7110 closes the spec's own documented GC-vs-fencing hazard; v8 is therefore the *minimum* substrate for this RFC | +| Durability waiters (fenced flush surfaces to waiters, not hangs) | v8 #7132 | **The ack point.** `ingest` resolves the client's request when the waiter resolves | +| MemTable + flushed memtables (Lance tables per generation, with pre-built indexes + bloom filter) | spec §MemTable, §Flushed MemTable | The fold input; generation order gives upsert correctness | +| `maintained_indexes` (memtable FTS/vector indexes; vector inherits base PQ/SQ params for distance comparability) | spec §MemWAL Index Details | Phase 2 fresh-tier search: FTS + vector queries stay indexed over unfolded data | +| MemWAL index: `merged_generations` **updated atomically with the merge-insert data commit**, concurrent-merger conflict resolution | spec §Index Details, Appendix 2 | The fold's exactly-once bookkeeping; two racing folds resolve via the conflicting commit's index, no progress regression | +| `index_catchup` progress | spec | Bridges base-table index lag after fold: reads use the flushed memtable's indexes for the gap instead of scanning | +| LSM merging read (`_gen`/`_rowaddr` dedupe), shard pruning, fresh-tier as-of cut, LSM FTS planner, LIMIT/OFFSET pushdown, cache prewarm | spec §Reader Expectations; v8 #7215, #7066, #7256, #7284 | Phase 2 fresh-tier read path — consumed, not reimplemented | +| GC contract (merged + index caught up + no retained version refs) | spec §Garbage Collector | Wired into `omnigraph cleanup`; the "retained version" clause maps to manifest-pinned versions | +| Staged transactions (`execute_uncommitted` for append/delete/merge-insert) | #6781 (merged) | The fold's preferred commit shape (§4.4) | +| `CommitBuilder` commit timeout; `skip_auto_cleanup` | v8 #6773; existing | Fold commit hygiene, same as every existing staged path | +| Tombstone-preserving point lookup; `delete_no_wait`; `Sealed` shard status (drop-table 2PC) | v9 #7482, #7483, #7361 | Phase 3: stream-mode deletes; clean disable/drop of an enrolled table | +| Unenforced primary key (immutable once set) | Lance 7 rule | Enrollment precondition — see §4.1 | + +Deliberately **not** consumed: Data Overlay files (v9-adjacent, voted +experimental 2026-06 — a different fast-write shape targeting in-place +overlays; watch-listed, not load-bearing here), MemWAL's multi-shard +horizontal scale-out (deferred to Phase 2 with sharding specs). + +## 4. Design + +### 4.1 Enrollment + +Stream ingest is **per node/edge type**, declared in the schema: +`@stream` annotation on the type. Schema apply records intent (no physical +work — same posture as `@index`). Enrollment's one physical precondition runs +at the first ingest write, not at apply: + +- The table's key column must carry Lance's unenforced-PK annotation (MemWAL + needs a PK for last-write-wins upsert and PK lookups). Under Lance ≥7 the + PK is **immutable once set**, so the enrollment step mirrors + `migrate_v1_to_v2`'s guarded shape exactly: key column already the PK → + no-op; no PK → set it; a *different* PK → loud refusal. +- The MemWAL index (sharding spec, `maintained_indexes`, writer config + defaults) is created through the existing index chokepoint discipline: + derived state, idempotent, never fails a logical op. + +### 4.2 Write path and ack + +A new explicit surface — `omnigraph ingest --stream` / `POST +/graphs/{id}/ingest` (NDJSON stream) — routes rows to a per-`(table, branch)` +`ShardWriter` held by the engine (warm, invariant 15). Per row, **before** the +WAL append, the synchronously-checkable validations run: type/shape, enum, +range/`@check`, required fields, defaults. These fail the row *before* +anything is durable — invariant 9 holds for everything checkable without a +base-table read. + +Referential integrity (edge endpoints), cardinality, and cross-row uniqueness +need base-table state and are validated **at fold time** (§4.4). A row that +fails there cannot be un-acked; it lands in a per-graph dead-letter table +(`_ingest_rejects`) with the typed error, surfaced via `ingest status` and the +server API — bounded and observable (invariant 13), never silent, and +documented as the stream-mode contract difference. This is the standard +streaming-system trade, stated honestly rather than hidden. + +Ack = the Lance durability waiter for the row's WAL entry resolving. Group +commit falls out of WAL-flush batching; the flush interval is the +latency/throughput knob and lives in `writer_config_defaults` so every writer +process shares it. + +### 4.3 Visibility tiers + +- **Default reads: unchanged.** A query reads its manifest snapshot + (invariant 3). WAL contents are not graph-visible until folded. Strong + consistency at graph-commit granularity, exactly as today. +- **Fresh-tier reads: explicit opt-in** (query-level annotation, e.g. + `read fresh`). The planner unions base-table-at-snapshot + flushed + memtables + (same-process) the live MemTable via Lance's LSM merging read. + Same-process this is read-your-writes (the spec's strong-consistency + condition: MemTable access + direct shard-manifest reads); cross-process it + is flushed-WAL-consistent (bounded lag = the unflushed MemTable). Both are + documented per-tier. This is invariant 6's explicit/auditable/non-default + clause used precisely as designed — not a silent fallback. + +### 4.4 The fold + +The fold is the MemWAL *merger* (spec §Background Job Expectations) run under +OmniGraph's commit discipline: + +1. Acquire the per-`(table, branch)` write queue (serializes against + interactive writers and other folds in-process). +2. Fold-time validation of pending generations (RI, cardinality, uniqueness + via `loader::composite_unique_key`) against the current snapshot + + generations-in-order; rejects → dead-letter, never a placeholder node + (deny-list). +3. Merge flushed generations in ascending generation order via merge-insert. + **Preferred shape:** `execute_uncommitted` (staged) → `commit_staged` → + one `ManifestBatchPublisher::publish` carrying the table version *and* the + graph-commit lineage rows — i.e. the fold is an ordinary OmniGraph writer. + **Open question Q1:** whether the MemWAL index's `merged_generations` + update (which the spec requires to ride the data commit atomically) is + expressible through the staged path. If not, Phase 1 runs the fold's merge + as an inline-commit residual under a `SidecarKind::IngestFold` recovery + sidecar — precisely the `optimize_indices` precedent — and migrates to + staged when upstream exposes it. +4. One fold = **one graph commit** (actor `omnigraph:ingest`, or the + enrolled writer's actor when single). Lineage lands in `__manifest` in the + same CAS as always. Fold triggers: generation count, byte size, max lag — + plus a synchronous `omnigraph ingest fold` for operators and tests. + +Crash windows: before WAL flush → row never acked, nothing durable. After +ack, before fold → WAL replays (spec §WAL Replay) under the next claimant's +epoch; acked data cannot be lost. Mid-fold → either the sidecar rolls the +residual forward (Phase 1 inline shape) or the staged state is invisible and +the fold re-runs; `merged_generations`' atomic update makes re-merge of an +already-merged generation impossible (spec Appendix 2). After fold, before +GC → LSM dedupe makes double-reads correct (spec §Reader Consistency). + +### 4.5 Branch interactions + +Phase 1 rule, chosen for smallness: **branch create/merge/delete on an +enrolled table first drives its WAL to quiescence** (fold-to-empty under the +same write queue). A branch never forks with an un-folded WAL tail, so branch +semantics are untouched. Relaxing this (per-branch WAL lanes) is a Phase 3+ +question; MemWAL shards are per-dataset-path, and branches are separate +version lineages, so the mapping exists — it is deferred, not blocked. + +### 4.6 Embeddings (`@embed`) interaction + +Embedding computation is external and slow; it cannot sit between the client +and the WAL ack. Stream-mode `@embed` columns are therefore computed **at +fold time** (the fold pipeline calls the embedding client on the folded +batch, same code path as `load`). Consequence: memtable-maintained *vector* +indexes cannot cover not-yet-embedded rows — fresh-tier vector search over an +`@embed`-sourced column sees rows only post-fold (fresh-tier FTS and scans +are unaffected). Documented per-column; RFC-015 owns the deeper embedding +pipeline. + +## 5. Reconciliation with Lance v8.0.0 (Phase 0 — the bump) + +v8.0.0 released ~2026-07-01 (RC3 vote closed). This RFC's Phase 0 is the +7.0.0 → 8.0.0 bump as its own PR with the full lance.md alignment-audit +stanza. Items already identified as load-bearing: + +- **MemWAL hardening is the reason v8 is the floor:** #7110 (fencing WAL + sentinel on claim — closes the spec's documented GC-fencing hazard, which + this RFC would otherwise inherit), #7132 (fenced flush surfaces to + durability waiters — without it our ack path can hang), #7215/#7066/#7284/ + #7256 (fresh-tier read planning this RFC consumes in Phase 2), #7054 + (HNSW params for memtable writers). +- **Blob compaction fix landed** (#7017: all `BlobKind` in blob-v2 + `compact_files`): flip `LANCE_SUPPORTS_BLOB_COMPACTION`, remove the + `optimize` skip branch — the `compact_files_still_fails_on_blob_columns` + surface guard turns red on the bump by design and forces this. +- **Critical merge-insert fix** (#7251: silently dropped matches when a + leading payload column is all-null): audit whether any existing OmniGraph + merge path could have hit it (all-null embedding columns in `LoadMode:: + Merge` are plausible); add a pinned regression either way. +- **Breaking index-segment migration** (#6869 bitmap→segment-based, #6997 + removed segment builder, #7013 BTree on the segmented framework): OmniGraph + builds BTREE/inverted/vector through one chokepoint — expect API churn + there; re-run every `lance_surface_guards.rs` guard first, per the standing + bump protocol. +- **FTS tokenizer default churn** (#6968 ICU default, then #7006 restored + simple): verify the effective default matches what OmniGraph's FTS contract + shipped (Hyrum's-law item — our search results' tokenization is observable + behavior). +- **#7158** (fail-fast casting for indexed columns) intersects schema apply; + **#6916** (index-accelerated filtered `count_rows`) and **#7129** (no + index-file listing after writes) should show up as free improvements in + `warm_read_cost.rs` / `write_cost.rs` — re-baseline, don't loosen budgets. +- **Namespace feature flags in `__manifest`** (#7191): dir-catalog-only, but + re-verify the v7 audit's conclusion that Lance's native `DirectoryNamespace` + stays decoupled from OmniGraph's manifest (that decoupling was contingent + on the legacy boolean PK key — confirm the contingency still holds under + v8's flag writes). +- Re-check the two still-open residual issues on the bump: #6666 (vector + index two-phase) — **still open**, `create_vector_index` stays inline; + #6658 shipped back in 7.0.0 (MR-A unchanged, still pending). + +## 6. Reconciliation with the Lance v9 line (beta) + +v9 is at beta.10; **not a production target until stable**. What it adds that +this RFC phases on: + +- **Stream-mode deletes:** tombstone-preserving point lookup (#7482) and + `ShardWriter::delete_no_wait` (#7483) give MemWAL delete semantics. Phase 3 + adopts them; until then stream ingest is insert/upsert-only and delete + remains an interactive-path operation (the D2 rule's scope is unchanged — + note MR-A may retire D2 on the interactive path independently). +- **Enrollment teardown:** `Sealed` shard status + `ShardWriter::abort` + (#7361, the drop-table 2PC fence — also the subject of the 2026-06 format + vote #7418) is the clean disable path for `@stream` removal. +- **#7468** (reject `defer_index_remap` with stable row IDs) touches the + future stable-row-id traversal plans (invariants.md known gap) — flag for + that workstream, not this one. +- **Data Overlay files** (#7401, voted experimental #7447): a different + fast-targeted-write primitive (masking overlays over base files). If it + matures it may fit *small in-place updates* better than WAL-upsert-fold; + watch-listed as a possible Phase 4 refinement, not a dependency. +- MemWAL fixes keep landing on v9 betas (#7489 cross-generation block-list on + in-memory scan arms) — confirming the experimental-spec churn risk (§10) + and the value of keeping our exposure transient-state-only. + +## 7. Composition with upcoming Lance multi-table commits + +Upstream state (verified 2026-07-02): issue #6668 ("Multi-dataset atomic +commit primitive") is **closed into RFC discussions**, not shipped. Two live +proposals: + +- **#7260 — batch commit record** (v2, supersedes #6775; authored by this + team — disclosure as in the upstream thread): stage per-table via + `execute_uncommitted` → publish **one immutable record** in a `_txn/` log + via put-if-not-exists → idempotent, reader-completable finalize. Namespace- + reader atomicity; exclusive commit authority per enrolled table (§3.3a + there). +- **#7264 — branching MTT** (authored by the Lance maintainer, jackye1995): + per-transaction shallow-clone branches, one atomic `__manifest` (catalog) + repoint as the commit, leased **barrier commits** to fence the per-table + fast path during rebase, riding #7263 (cross-branch rebase) and #7185 + (UUID branch paths). Notably, its Alternative A (fast-forward-main instead + of pointer-swap) is OmniGraph's current architecture verbatim — N dataset + commits + a write-ahead intent record + idempotent roll-forward recovery — + so one plausible upstream outcome is Lance adopting the sidecar shape this + codebase already runs. + +Both keep the settled invariants (#7222/#7176): physical files are the +eventual source of truth; put-if-not-exists is the primitive; no per-commit +work on a catalog table. Neither is in v8 or the v9 betas. + +**How this RFC composes — the one design rule.** The fold (and every other +writer) reaches publication only through the `ManifestBatchPublisher` seam. +That is already true and this RFC keeps it true. Consequences: + +- **Under a #7260-shaped primitive:** the fold's N staged table commits + the + manifest publish collapse into operations of one batch record. The residual + Lance-HEAD-before-manifest gap — the entire reason recovery sidecars exist + (invariant 5) — disappears *by construction*, and the sidecar machinery + (including this RFC's possible `SidecarKind::IngestFold`) becomes removable + scaffolding. This is exactly the degrade-to-no-op shape invariants.md says + to prefer. OmniGraph's `__manifest` remains as the *graph-semantic* layer + (lineage rows, branch heads, snapshot pinning) — the batch record is a + commit mechanism, not a lineage store; graph_commit rows simply become one + more operation in the batch, keeping lineage atomic with visibility as + today. Adoption = swapping the publisher's internals; WAL, fold logic, and + read paths are untouched. +- **Under a #7264-shaped primitive:** OmniGraph's manifest *is* the catalog + in that RFC's vocabulary, and OmniGraph already runs the consumer-side + pattern (lazy per-write forks ≈ transaction branches; publish CAS ≈ the + atomic repoint). The genuinely new upstream piece — the **leased barrier + commit** — is independently valuable to OmniGraph: it is a concrete + candidate for the cross-process serialization primitive that two documented + known gaps explicitly wait on (recovery-sweep-vs-foreign-live-writer; + cross-process fork reclaim). One caveat, already raised on the upstream + thread (ragnorc, 2026-06-13): a TTL lease alone cannot make "barrier held + + tip unmoved + repoint" atomic — if the critical section outruns the lease, + a competing writer breaks the barrier and advances the tip, and the + coordinator's repoint silently loses that commit — so the barrier needs an + epoch/generation, not just a TTL. Any OmniGraph adoption inherits that + requirement. If #7264 lands, adopt the (epoch-carrying) barrier for those + gaps even if the fold never uses MTT directly. + **Update (2026-07-05):** the 2026-07-03 comment on #7264 proposes the + unification directly — detached-commit staging + the #7260 record as the + intent + fast-forward-main promotion, prototype-validated (24-test probe + suite). The barrier is now a slot-occupying content-identical commit + (epoch in the record log, enforced in the writer's commit path), answering + the TTL-lease caveat above structurally. Measured: ~450–600 ms per + transaction, ~6–7 txn/s on S3 with **group commit as the lever** — i.e. + the same amortization shape as this RFC's fold, strengthening Phase 4's + "publisher swap, not redesign" claim with numbers. The record log also + carries a durable pruned-through GC boundary (advance-before-delete, + writer re-verifies after put) — see open question 4. +- **MemWAL + MTT together:** the fold's merge commit, its `merged_generations` + index update, and the manifest publish could all join one batch — closing + Q1 (§4.4) from above rather than below. + +The WAL is upstream of the commit point in every scenario; the commit point's +mechanics can change under it freely. That orthogonality is the payoff of +constraint 1 (§0) and the reason this RFC refuses any design where WAL state +participates in publication authority. + +## 8. Invariants and deny-list walk + +- **1 Respect the substrate:** MemWAL consumed as spec'd; zero WAL code owned + (§3). ✅ +- **2 Manifest-atomic visibility:** WAL data becomes graph-visible only via + the fold's single publish; fresh-tier is explicitly outside graph + visibility. ✅ +- **3 One snapshot per query:** unchanged; fresh-tier unions the snapshot + with WAL tiers *at plan time*, never re-reads head mid-query. ✅ +- **4 One publish boundary per mutation:** the fold is one boundary. ✅ +- **5 Recovery coverage:** fold Phase 1 residual (if Q1 forces inline) gets + `SidecarKind::IngestFold`; WAL-side recovery is Lance's replay + fencing. ✅ +- **6 Strong consistency default:** ack = durable; default reads unchanged; + fresh-tier is the invariant's own explicit/read-only/auditable/non-default + clause. ✅ +- **7 Indexes derived:** memtable maintained indexes + `index_catchup` are + Lance's implementation of this invariant; `@embed` vector gap documented + (§4.6). ✅ +- **9 Loud integrity:** sync-checkable validation pre-ack; fold-time RI to a + dead-letter table with typed errors — a *documented contract difference*, + not a silent weakening; no placeholder nodes ever. ⚠️ documented +- **13 Bounded/observable failures:** dead-letter + `ingest status` + fold + lag metrics on the observability surface. ✅ +- **15 One source of truth, cheaply derived:** ShardWriter handles held warm; + shard-manifest `version_hint` is the cheap probe; WAL state is transient + (folded + GC'd), so nothing long-lived can drift. ✅ +- **Deny-list sweep:** no custom WAL (Lance's); no acks before durability; + no per-table publish outside the publisher (fold uses it); no job queue for + manifest-derivable state (the fold is a reconciler over MemWAL state, which + is NOT manifest-derivable — it is upstream input, the one shape the rule + permits); no silent eventual consistency (explicit tier); no raw FS I/O + (WAL files are Lance-managed inside the table dir). ✅ + +## 9. Testing plan (per testing.md) + +- **Surface guards** (`lance_surface_guards.rs`): pin ShardWriter claim/ + append/flush shapes, durability-waiter semantics, `merged_generations` + atomic-update behavior, tombstone APIs when v9 lands. Bump protocol + unchanged: guards run first. +- **Failpoints:** WAL-append failure → no ack, zero durable state; crash + after ack before fold → replay recovers the row (the acked-implies-durable + oracle); fold crash windows (each side of `commit_staged`/publish, sidecar + lifecycle if inline); two folds racing via `Rendezvous` → exactly-once via + `merged_generations`; fencing: zombie writer post-claim cannot ack. +- **DST:** concurrency cells ride PR #318's `Cohort` multi- + coordinator harness (in-process and two-process tiers) — two coordinators + ingesting + folding the same graph must converge to one linear chain; + extend the op alphabet with `ingest` + `fold` ops; model gains a + "pending" row set; oracles: acked rows survive any crash, fold equivalence + (model == graph after fold), fresh-tier read-your-writes in-process; + `FaultAdapter` covers WAL object I/O; S3 battery cell for the WAL path. +- **Cost budgets** (`helpers::cost`): ack-path object-store ops O(1) and flat + in history *and* in WAL depth; fold cost bounded by generations folded + (working set), never by table history; extend `write_cost.rs` + + `write_cost_s3.rs` (the opener term is S3-only, per the backend-split + note). +- **CLI parity:** `ingest --stream` in `parity_matrix.rs` (embedded vs + remote) and a DST cross-backend cell. + +## 10. Risks + +- **MemWAL is experimental upstream** (spec banner; live format votes — + #7418 Status field mid-2026-06). Mitigation is structural: WAL state is + *transient* (folded then GC'd), so a format change between Lance versions + can be handled by fold-to-quiescent before the bump; no long-lived on-disk + state depends on the MemWAL format. This must stay true — resist any + future temptation to park durable state in WAL form. +- **Q1 (staged fold)** may force the inline-residual shape initially — a + known, sidecar-covered, deny-list-documented pattern with two precedents. +- **Dead-letter semantics** are a real contract change for stream mode; + user-docs must present it before the endpoint ships (maintenance contract + rule 1). + +## 11. Phasing + +| Phase | Deliverable | Gate | +|---|---|---| +| 0 | Lance 7.0.0 → 8.0.0 bump: full alignment audit stanza, guards re-run, blob gate flipped, #7251 audit | v8.0.0 (released) | +| 1 | `@stream` enrollment; single-shard ShardWriter per (table, branch); sync validation; ack on durability waiter; fold (staged if Q1 allows, else sidecar-covered inline) publishing one graph commit; manifest-tier reads only; dead-letter + `ingest status`; branch-ops fold-to-quiescent | Phase 0 | +| 2 | Fresh-tier reads (explicit opt-in); `maintained_indexes` (FTS + vector, `@embed` caveat); multi-shard via sharding specs; group-commit tuning | Phase 1 | +| 3 | Stream-mode deletes (tombstones); `@stream` removal via Sealed/abort; per-branch WAL lanes exploration | Lance v9 stable | +| 4 | Adopt the multi-table commit primitive (#7260/#7264 outcome): publisher-seam swap, retire fold sidecars; evaluate barrier commits for the cross-process known gaps | upstream vote/ship | + +## 12. Open questions + +1. **Q1:** Can `merged_generations` ride `execute_uncommitted` + + `commit_staged`, or is the fold's merge inline-only today? (Determines + Phase 1 shape; upstream question — possibly a #6781-shaped PR we offer.) +2. Fold-time RI rejects: dead-letter only, or optional strict mode where a + fold halts (back-pressuring ingest) on the first reject? +3. Actor granularity in fold lineage: single ingest actor vs. per-WAL-entry + actor aggregation into commit metadata. +4. `cleanup` integration ordering with the Q8 cleanup-resurrection watermark + (the internal-tables GC gap) — does WAL GC land before or after it? + **Note (2026-07-05):** the #7264 record-log design carries exactly the + durable GC-boundary primitive `iss-cleanup-boundary-watermark` asks for + (GC advances the boundary before deleting; a writer re-verifies + `seq > boundary` after a successful put). If that ships, the watermark + arrives as substrate rather than omnigraph protocol — factor into the + build-vs-wait decision on that P1. +5. Should fresh-tier be exposed over HTTP in Phase 2, or engine/CLI-only + until the consistency-tier docs mature? diff --git a/docs/rfcs/rfc-0019-heads-and-fences.md b/docs/rfcs/rfc-0019-heads-and-fences.md new file mode 100644 index 00000000..2319706e --- /dev/null +++ b/docs/rfcs/rfc-0019-heads-and-fences.md @@ -0,0 +1,499 @@ +--- +type: spec +title: "RFC-019 — Heads and Fences: structural O(1) writes without a warm-cache truth fork" +description: Replaces the warm-publish/pinned-open machinery of PR #318 with two structural changes — durable per-table head rows ("refs, not replay") and substrate-native key-conflict fencing (Lance's unenforced-PK KeyExistenceFilter) — landing together as internal schema v5; composes with RFC-018 (WAL ingest) and the upstream multi-table-commit direction. +status: draft +tags: [eng, rfc, write-path, manifest, lance, omnigraph] +timestamp: 2026-07-04 +owner: +--- + +# RFC-019 — Heads and Fences + +**Status:** Draft / for discussion +**Date:** 2026-07-04 +**Surveyed:** omnigraph `main` @ 98530a0e (0.8.0); Lance pinned 7.0.0 (+ vendored lance-table carrying lance#7480); upstream Lance v8.0.0 (released 2026-07-01), v9.0.0-beta.15; PR #318 at `2aab48ba` (reviewed 2026-07-04, 8 verified findings) +**Companion docs:** RFC-018 (streaming-ingest WAL), PR #318's plan doc (`unlimited-history-latency-plan.md`, whose §9 "U2" this RFC promotes from follow-up to prerequisite) +**Audience:** OmniGraph maintainers + +--- + +## 0. TL;DR + +PR #318 makes writes O(1) in commit-history depth by **caching the manifest fold** +(warm publish) and **pinning staging opens at the transaction base** (list-free +opens). Both wins are real; both mechanisms buy speed with a permanent +correctness liability: a second in-memory truth path that must be proven +equivalent to the on-disk one forever, and a widened window for the known +silent-duplicate-key bug (MR-714). + +This RFC proposes getting the same wins **structurally**, in one storage-format +migration (internal schema v4 → v5, "heads and fences"): + +1. **Head rows** (*refs, not replay*): one mutable `table_head:` row + per table in `__manifest`, updated in the same atomic merge-insert as the + journal rows — the exact mechanism `graph_head:` already uses. + "What's current?" becomes a direct O(tables) filtered read. There is no fold + left to cache, so the warm/cold dual path is unnecessary rather than + defended. +2. **Fence-first** (*substrate-native key conflicts*): annotate every data + table's key columns as Lance's **unenforced primary key** and route data + merges onto the v2 plan path, engaging Lance's `KeyExistenceFilter` + conflict detection (shipped upstream in PR #5633, 2026-01): two concurrent + merge-inserts of the same key produce a **retryable conflict**, never a + silent duplicate. MR-714 closes as a class. Once fenced, pinned opens are + safe by construction and may return for their residual ~1-RPC saving. + +Sequenced: Lance 7→8 bump (fence-first requires v8's #7251 fix; also +RFC-018's Phase 0) → v5 migration (both format changes in one stamp bump, one +upgrade story) → RFC-018's WAL phases on top. What survives from #318: the +freshness probe, session-held handles, the DST `Cohort` harness, the cost +gates — and warm publish itself **as an explicitly temporary stopgap with v5 +as its named demolition**, if the latency is needed before v5 lands. + +Along the way, §2 proposes a **shaping amendment to invariant 15**: derived +views of the *immutable past* may be cached in memory freely; derived views +of the *mutable tip* belong durably inside the source of truth, and may live +in process memory only as hints arbitrated per use by the commit authority. + +The one-line thesis: **heads make state cheap to read, fences make races +loud, the WAL (RFC-018) makes commits rare where visibility can wait — and +the single publish seam underneath keeps all three swappable for whatever +multi-table primitive Lance ships.** + +--- + +## 1. Motivation: the case against warm-cache permanence + +This section argues against **#318 as the endgame**, not against merging its +branch: §7 states exactly which parts survive. The arguments, in descending +order of weight: + +### 1.1 The root cause is the representation, not the read + +`__manifest` accumulates a row per table-version per commit, so answering +"what's current?" requires folding O(commits) rows into an O(tables) answer. +#318 doesn't fix that — it **caches the fold**, and then spends enormous +effort defending the cache: version-coupled folds that fail closed on +`landed ≠ base+1`, warm/cold equivalence tests +(`warm_publish_rejects_duplicate_tombstone_like_cold`), a 20-line doc comment +explaining why two caches can't drift, a freshness probe, and a *second* +freshness mechanism (`warm_head_hint`) for a *second* cache (the commit +graph), coupled to the first by a parenthetical comment. When a correctness +story needs this much prose, the design is telling you something. + +### 1.2 It forks the truth path, permanently + +Warm and cold are two implementations of the same question, and every future +publisher change now carries a "and what does the warm arm do?" audit — +forever. The 2026-07-04 review of #318 is the exhibit: + +- the warm arm **already skips the per-branch stamp gate** the cold arm runs + (`refuse_if_stamp_unsupported` — confirmed finding, one-line fix); +- the duplicate-version guard **already diverges** between arms (latest-only + vs full-history; verified safe, but only after an adversarial verifier + wrote three paragraphs proving monotonicity and handoff shapes); +- `Warm(None)` + a lineage intent **already compiles into a parentless + commit**, saved only by a genesis invariant living two modules away — the + PR's "unrepresentable, not guarded" claim does not cover its own newest + seam. + +The repo's own "what does this look like after 5 more changes" test arguably +fails: each new warm field threads through **four layers** (`publish_inputs` +tuple → `WarmAttempt` borrow → `PublishPlan` lifetime → `from_warm` +re-clone — which also double-clones every map per write, defeating the +design's own "bundling allocates nothing" claim). + +### 1.3 It's misaligned with where the substrate is going + +Lance v9 is shipping session-cached manifest reuse (#7576, beta.12), stale +cached-manifest recovery (#7542), and single-flight index opens (#7464) — the +substrate is learning to keep manifests warm **itself**. Meanwhile MemWAL, +#7260 (batch commit record), and #7264 (catalog repoint; see the 2026-07-03 +unification comment proposing detached commits + the batch record) all +converge on the same idea: **"current state" should be one cheap +authoritative object, not a fold you have to cache.** Building bespoke +warm-state machinery one layer up duplicates what the substrate is about to +provide — the exact smell invariant 1's second clause warns about +("re-deriving per call what the substrate keeps warm is a substrate violation +even when no code is reimplemented" — and so is re-*building* it). + +### 1.4 The tell is in the PR's own plan + +#318's roadmap contains **U2: the head-pointer format change (internal schema +v5) — mutable per-table "latest" rows**. That *is* the fix for the +row-accumulation wall. If U2 makes a cold read O(tables), the warm machinery +mostly evaporates. Shipping ~4k lines of cache-defense *before* the +representation fix is building scaffolding the plan already schedules for +demolition — without naming the demolition. + +### 1.5 The pinned open bought ~1 RPC with a silent-corruption window + +Two facts sharpen the pinned-open half. First, the expensive opener was +already dead before #318: `main` carries RFC-013 step 3a +(`open_dataset_head_for_write` routes through the direct opener; the +namespace builder's O(depth) chain resolution is gone). Second, Lance's +latest-resolution is already cheap: `resolve_latest_location` is a local read +on FS, a version-hint GET + probe on non-lexicographic stores, and **one +bounded LIST** on S3 Standard (descending naming: first result is latest). +So pin-at-base saves roughly **one cheap RPC** per non-strict staging open +over live-HEAD-via-probe — and pays for it by widening the MR-714 +dup-`@key` window from [stage → commit] to [txn-capture → commit], with +every fence verifiably absent (queue acquired only at `commit_all`; CAS pins +refreshed to live; Lance's rebase is fragment-level; `@key` validation is +intra-delta by design). One RPC is not a good price for a wider silent- +corruption window. §4 removes the window instead. + +--- + +## 2. Invariant shaping: two kinds of derived views + +The strongest defense of #318's warm publish cites **invariant 15** and the +read path's precedent: "hold a warm derived view, refresh with a cheap probe, +never rebuild from cold storage per call" is the repo's constitution, the +query-latency work proved it on reads, and the write path was the last cold +re-deriver. Examining why that license does not transfer to writes turns out +to sharpen the invariant itself — this section derives the distinction and +proposes the amendment. (This is invariant *shaping*, not a dispute with the +invariant: both of its named failure modes are real and stay; what it lacks +is a scope qualifier its own author would recognize.) + +### 2.1 Why the read-path license doesn't transfer to write inputs + +Three asymmetries separate the paths: + +1. **Reads cache the immutable past; warm publish caches the mutable tip.** + The query-latency work's warm state is keyed by *version*: handles at + `(table, branch, version, e_tag)`, snapshots pinned at a resolved version. + Version-pinned data never changes under its key, so a cache-key discipline + is the entire correctness story — no equivalence proofs needed. Warm + publish caches the *moving frontier* (current versions, registrations, + tombstones, the lineage head). Caching an immutable past and caching a + mutable present are different problems that share the word "cache." +2. **Staleness is tolerated by contract on reads and forbidden on writes.** + Invariant 3 makes a query's slightly-old snapshot *semantically correct + output* — a warm view lagging by a probe window delivers exactly the + promised semantics. A publish is the opposite: its inputs must be exact + against live state or the commit is wrong, so every staleness a read + shrugs off, a write must *detect perfectly*. The fail-closed fold + coupling, the warm/cold equivalence tests, and the second freshness + mechanism in #318 are not incidental — they are the cost of moving a + staleness-tolerant pattern into a staleness-intolerant context. The + precedent's cheapness does not transfer because the cheapness came from + the tolerance. +3. **The blast radius inverts.** A wrong warm read self-heals at the next + probe and corrupts nothing. A wrong warm publish is *durable*: a + mis-parented commit, a skipped gate, an overwritten guard — persisted + state, discovered later. Same mechanism, categorically different failure + cost. + +There is also a textual point: invariant 15's first named failure mode is "a +*parallel copy* the engine maintains can drift" — and an in-memory copy of +the mutable tip consumed as commit input *is* that parallel copy. The "hold +warm, cheap probe" blessing was written from read-path experience with +version-pinned views. Read strictly, the invariant is *better* satisfied by +head rows: a durable materialized view maintained **inside** the source of +truth, updated atomically with it — drift not defended against but +impossible. + +What the precedent legitimately proves: the probe/incarnation machinery is +sound, and the team can operate such a cache. Feasibility and permission — +not appropriateness for commit inputs. (One nuance in #318's favor: the +coordinator's folded `known_state` already existed for read-snapshot +resolution; the PR mostly lets *writes* consume a cache reads already +maintained. That is precisely the trap — the marginal code is small while +the marginal correctness class changes completely.) + +### 2.2 Proposed amendment to invariant 15 + +Append to the invariant's statement in `docs/dev/invariants.md`: + +> **Derived views come in two kinds, and the license differs.** +> *Views of immutable, version-pinned state* (a snapshot at a resolved +> version, a handle at `(table, version, e_tag)`, a topology index over +> pinned tables) may be held warm in process memory freely: the source +> cannot change under them, so a cache key is the whole correctness story. +> *Views of the mutable tip* (current versions, registrations, heads) held +> in process memory are the "parallel copy that can drift" this invariant +> forbids — **when consumed as write input**, staleness is not a degraded +> answer but a wrong commit. The tip's derived view belongs *inside* the +> source of truth as a durable row maintained atomically with it (the +> `graph_head` pattern), where drift is impossible rather than defended. An +> in-memory tip copy is acceptable only as a *hint* whose every consumption +> is arbitrated by the commit authority — never as an input whose +> correctness rests on call-site discipline. + +### 2.3 Proposed deny-list addition + +> - Warm in-memory state of the mutable tip consumed as publish/commit +> input, where correctness depends on invalidation discipline rather than +> per-use arbitration by the commit authority. Cache the immutable past; +> materialize the mutable present durably. + +### 2.4 Process and consistency check + +Per invariants.md's own maintenance policy, changing an invariant takes the +same review as code; this RFC is the vehicle, so the rule change and the +design it licenses are reviewed as one unit. The amendment's consistency +check is that one rule yields three coherent verdicts: it retroactively +**validates** the read-path work (immutable-view caching, exactly as +licensed), **reclassifies** #318's warm publish (a tip copy as commit input +— the forbidden shape, permitted only as a stopgap *hint* under CAS +arbitration with a named demolition, per §7), and **mandates** head rows as +the end state (the mutable present, materialized durably). One rule, three +verdicts, no contradictions — which is what an invariant is for. + + +## 3. Head rows (*refs, not replay*) + +### 3.1 Design + +Add one **mutable head row per table** to `__manifest`: + +- `object_id = table_head:` (same keying discipline as + `graph_head:`), carrying the table's current `table_version`, + location, and branch. +- Written by `ManifestBatchPublisher` in the **same merge-insert commit** as + the immutable journal rows and lineage rows — pointer and journal cannot + diverge because they land atomically (the RFC-013 Phase 7 argument, reused). +- The journal rows are unchanged: time travel, `snapshot_at_version`, and + diff/change feeds keep reading immutable history exactly as today. + +"Current state" for a publish or a snapshot resolve becomes a **filtered read +of O(tables) head rows** — no fold, no O(commits) term, cold is fast. The +mental model is git's: the log is replayed for history, never for "where is +main"; `refs/` answers that directly. `graph_head:` proved the +mechanism; head rows generalize it to every table. + +### 3.2 What it deletes + +- The warm/cold dual in the publisher (`WarmAttempt`, `LoadedPublishState:: + from_warm`, `LineageSource::Warm`, the version-coupled fold's fail-closed + arm) — there is no fold to cache. +- The `warm_head_hint` second-freshness mechanism — the lineage head is a + head row read in the same filtered read. +- The warm/cold equivalence test surface and the four-layer field-threading. + +What it keeps needing: the freshness probe (one cheap incarnation check +before trusting any held handle) and the publisher CAS — both unchanged. + +### 3.3 Cost and constraints + +- **Storage-format change** → internal schema stamp bump (v4 → v5) under the + strand model: old binaries refuse new graphs cleanly; upgrade is the + documented rebuild-or-migrate story. This is the deliberate, earned, + *reversible-never* change — hence RFC rather than measurement (the repo's + reversibility rule, applied in the direction it actually points here). +- Head rows are hot rows: every publish rewrites O(touched tables) of them + via `WhenMatched::UpdateAll` — the same write shape `graph_head` already + exercises at every commit; no new contention point (the per-branch + serialization remains the `graph_head` row, §7.1 of RFC-013). +- `__manifest` compaction (already in `optimize`) keeps the journal bounded; + head rows make the *hot path* independent of whether compaction ran — + removing the operational assumption that was #318's strongest argument for + the cache. + +## 4. Fence-first (*substrate-native key conflicts*) + +### 4.1 What Lance already ships (verified against source, 2026-07-04) + +- **The key checker exists since January** (PR #5633): a merge-insert whose + ON columns match the schema's **unenforced primary key** attaches a + `KeyExistenceFilter` (split-block bloom of *inserted* keys) to its + `Operation::Update` transaction (`exec/write.rs`: `is_primary_key` + gating). +- **At commit, the conflict resolver intersects filters** of concurrent + transactions (`conflict_resolver.rs:344-369`): overlap → **retryable + conflict**; mismatched filter configs → conflict (safe default); a + concurrent bare `Append` → conflict (conservative). Silent duplicates are + structurally impossible for fenced tables. +- **OmniGraph already uses this — for metadata only.** `__manifest.object_id` + carries the unenforced-PK annotation and the publisher's merge runs + `use_index(false)` (the v2 plan path), which is exactly why the manifest + CAS enjoys row-level conflict detection. Data tables get neither. +- **Two gates, both currently closed for data tables:** (a) no PK annotation + on data tables; (b) data merges run the **legacy Merger** (a BTREE on the + join key disables `can_use_create_plan`), and the legacy path hardcodes + `inserted_rows_filter: None` ("not implemented for v1") in v7, v8, **and + v9-beta.15**. + +### 4.2 Design + +1. **Annotate data-table keys as the unenforced PK** (`id` for nodes, + `src`+`dst` composite for edges) at dataset creation. The PK is immutable + once set (Lance 7 rule), so for existing graphs this is a format moment — + which is why it rides the same v5 stamp bump as head rows (§6). +2. **Route data merges onto the v2 plan path** so the filter is actually + produced. Two options, decided by measurement in Phase 1: + - **(a) `use_index(false)` on data-table merges** (what the publisher + already does for `__manifest`): the join becomes a hash join instead of + a BTREE probe. Cost profile changes for large tables — needs a + `write_cost` measurement, not an assumption. + - **(b) Upstream ask: implement `inserted_rows_filter` on the indexed + (legacy) path** — a #6806-shaped contribution; the filter builder is + path-independent, the legacy Merger simply never wired it. + Option (a) unblocks immediately on v8; option (b) removes (a)'s cost + caveat later. Do (a), file (b). +3. **Writer-side conflict handling:** a fenced conflict surfaces as Lance's + retryable conflict; the staged-write path maps it to the existing + `ExpectedVersionMismatch`/retry vocabulary (non-strict ops retry the + merge against refreshed state under the write queue — the join re-runs + and the insert becomes an update; strict ops already 409). The user-visible + contract: concurrent same-key upserts **serialize or retry loudly** — + MR-714's dup-`@key` allow-list entry in the DST oracle flips from "known + open bug" to a hard failure. + +### 4.3 Prerequisite: Lance v8 + +The v2 plan path with a partial or racing source is only trustworthy on +≥ v8.0.0 (#7251 fixed the silent match-dropping under an all-null leading +column on the indexed/legacy boundary). The 7→8 bump is therefore Phase 0 — +shared with RFC-018, carrying the standing alignment-audit protocol, and +re-vendoring the lance#7480 `lance-table` patch at 8.0.0 (the fix ships in no +release ≤ 8.0.0). + +### 4.4 Pinned opens, reconsidered + +With the fence on, pin-at-base staging opens become **safe**: the race they +widen produces a conflict, not a duplicate. The residual ~1-RPC saving can +then be taken freely. Until the fence is on, non-strict staging opens should +stay at step-3a live-HEAD-via-cheap-probe (§1.5) — the pin's price is wrong +only while the window is silent. + +## 5. Relationship to RFC-018 (WAL) — the third leg + +Head rows and fences fix the **interactive** path (ack = visible): fast state +read, fenced staged commit, one CAS — near the physical minimum for +visibility-at-ack semantics. The **per-commit floor** (round trips, the +`graph_head` CAS rate) is real but is the *price of the semantics*, and only +workloads that can defer visibility can escape it. That is RFC-018's WAL: +ack on durability, fold N writes into one commit **through the same publish +seam**. Notably the fold's fenced merge benefits from §4 directly (MemWAL +requires the PK anyway — RFC-018 §4.1's enrollment step becomes a no-op once +v5 has annotated all tables), and head rows make the fold's publish cheap. +The three pieces are one program: + +| Piece | Kills | Semantics | +|---|---|---| +| Head rows | O(history) manifest fold per write | unchanged | +| Fence-first | silent dup-`@key` under races (MR-714) | races become loud retries | +| WAL + fold (RFC-018) | per-commit floor, for deferrable-visibility workloads | ack = durable; visible at fold (opt-in surface) | + +**Upstream addendum (2026-07-05).** The lance#7264 discussion's 2026-07-03 +comment turned the Phase-4 target concrete: a unification of the two upstream +MTT proposals — **detached commits as staging** (durable, reader-invisible, +contention-free), the **#7260 batch record as the write-ahead intent** (one +put-if-not-exists in `_txn/` as the commit point), and **fast-forward-main +promotion** (`Restore{staged_tip}` onto each table's single monotonic chain, +roll-forward completable by anyone, idempotently) — validated by a 24-test +probe suite against Lance main. Three points matter here: (1) the barrier +evolved from a TTL lease to a **slot-occupying content-identical commit at +N+1** with the epoch in the record log, enforced in the writer's commit path — +the fence-first shape (§4's philosophy), safety from commit-slot mechanics +rather than clock discipline; (2) measured: ~450–600 ms commit, ~6–7 txn/s on +S3 independent of catalog size, group commit as the lever — the numbers behind +this RFC's Phase-4 swap and RFC-018's fold; (3) the `_txn/` record log carries +a **durable pruned-through GC boundary** (GC advances it before deleting; a +writer re-verifies `seq > boundary` after its put succeeds) — the +`iss-cleanup-boundary-watermark` fix shape arriving as a substrate feature. +Head rows compose unchanged: `__manifest` remains the graph-semantic layer +(lineage, branch heads, snapshot pins); record+promote replaces the +publisher's internals, not the graph's refs. + +## 6. Migration: internal schema v5 ("heads and fences") + +One stamp bump, two format changes, one upgrade story: + +- `__manifest` gains `table_head:*` rows (written at migration for every + live table; thereafter by every publish). +- Every data table's key columns gain the unenforced-PK annotation. New + tables: at creation. Existing tables: set-if-absent, loud refusal on a + *different* existing PK (the `migrate_v1_to_v2` guarded shape, per + RFC-018 §4.1's identical need — coordinate so v5 satisfies both RFCs). +- Old binaries refuse v5 graphs at open with the standard stamp message; the + upgrade path is the versioning policy's documented one. **This also + retro-fixes #318's outstanding stamp-bump debt** (its lineage-row format + change shipped stamp-silent — review finding #1) if #318 merges first: + fold that correction into v5. + +## 7. Disposition of PR #318 + +Survives, unconditionally: the freshness probe and incarnation machinery; +session-held dataset handles (aligned with Lance #7576); the `Cohort` +multi-coordinator DST harness and cross-process suites; the cost-gate +extensions (`write_cost` served-regime gates); RFC-013 step 3a's direct +opener; the lineage read-back-from-`_row_created_at_version` idea (with the +v5 stamp bump it was owed, and a compaction surface guard). + +Survives as a **named stopgap** (if interactive latency is needed before +v5): warm publish — merged only with (a) the stamp bump it owes, (b) the +warm-arm stamp-gate fix and the `Warm(None)` fail-closed guard, (c) an +explicit demolition note: *v5 head rows retire `WarmAttempt`*. If the team +won't commit to (c), §1 argues the machinery shouldn't merge at all. + +Retired by this RFC: pin-at-base for non-strict staging (until §4's fence is +on), `warm_head_hint`, the warm/cold equivalence surface. + +## 8. Invariants walk (abbreviated) + +- **1 Respect the substrate:** consumes `KeyExistenceFilter`, + `resolve_latest_location`, and the PK machinery instead of parallel + omnigraph mechanisms; removes a warm shadow of manifest state. ✅ +- **2 Manifest-atomic visibility:** head rows ride the same single publish + CAS; nothing publishes outside the publisher. ✅ +- **6 Strong consistency default:** unchanged; fences convert a silent + anomaly into a loud retry — strengthening, not relaxing. ✅ +- **9 Loud integrity / deny-list "no silent failures":** MR-714's silent + duplicate becomes structurally impossible on fenced tables. ✅ +- **15 One source of truth, cheaply derived (as amended by §2):** head rows + *are* the cheap derivation, held durably in the single source rather than + shadowed in process memory — the mutable tip materialized where drift is + impossible, exactly the shape the amendment mandates. ✅ + +## 9. Testing plan (per testing.md) + +- **Surface guards:** pin `KeyExistenceFilter` production on the v2 path for + PK-annotated tables; pin the conflict-resolver overlap→retryable-conflict + behavior; pin (and re-pin at each bump) the legacy path's "not implemented + for v1" so option 4.2(b)'s arrival is noticed; pin head-row schema shape. +- **Engine:** same-key concurrent upsert cell flips from DST allow-list + (MR-714) to a hard assertion: both writers succeed serially or one + retries — never two live rows. Extend `writes.rs` with the + fenced-conflict retry path; extend `merge_truth_table` if merge semantics + observe conflicts. +- **Cost gates:** `write_cost.rs` — publish reads O(tables) head rows flat + in history (the head-row replacement for the warm-publish gate); + `use_index(false)` merge cost measured at row-count depth before choosing + 3.2(a) permanently. +- **DST:** the same-key interleave cell (currently allow-listed) becomes the + fence's regression net; cross-process cells assert conflict-not-dup. +- **Migration:** v4→v5 migration tests both format changes + refusal + matrix (old binary/new graph, new binary/old graph). + +## 10. Phasing + +| Phase | Deliverable | Gate | +|---|---|---| +| 0 | Lance 7→8 bump (shared with RFC-018 Phase 0): alignment stanza, guards re-run, re-vendor lance#7480 patch at 8.0.0, #7251 regression pinned | v8.0.0 (released) | +| 1 | Fence-first on new tables: PK annotation at creation + `use_index(false)` data merges (measured); conflict→retry mapping; file upstream ask 4.2(b); revert pin-at-base to step-3a live-HEAD opens | Phase 0 | +| 2 | v5 "heads and fences" migration: head rows + PK backfill on existing graphs, one stamp bump (folding in #318's owed bump if merged); publisher reads head rows; retire `WarmAttempt`/`warm_head_hint` | Phase 1, RFC review | +| 3 | Re-enable pin-at-base opens (now fenced) if the ~1 RPC matters; RFC-018 WAL phases proceed on the v5 substrate | Phase 2 | +| 4 | Upstream multi-table primitive — now concretely the #7260×#7264 record+promote unification (see §5 addendum): publisher-seam swap; recovery sidecars retire by construction; head rows remain the graph-semantic layer | upstream ship | + +## 11. Open questions + +1. **4.2(a) vs (b):** what does `use_index(false)` cost on large-table + merges (hash join vs BTREE probe)? Measure before committing; (b) is the + clean end state. +2. **Edge composite PK:** Lance composite unenforced PKs — verify + `src`+`dst` (+`id`?) matches the merge ON columns for edge tables, or key + edges on `id` alone and fence src/dst moves differently. +3. **Head-row read shape:** filtered read (`object_id IN (...)`) vs a + dedicated head partition — measure at catalog width. +4. **Migration mechanics under the strand model:** in-place v4→v5 migration + (one guarded pass, like `migrate_v1_to_v2`) vs export/import — the PK + annotation is in-place-able; head-row backfill is one publish; propose + in-place with the standard guarded shape. +5. **Does #318 merge first?** This RFC is written to compose either way + (§7); the sequencing decision is the team's, but the stamp-bump debt must + land with whichever ships first. diff --git a/docs/user/operations/upgrade.md b/docs/user/operations/upgrade.md index 2243c2a8..1bdcc849 100644 --- a/docs/user/operations/upgrade.md +++ b/docs/user/operations/upgrade.md @@ -12,16 +12,31 @@ changing) is in [docs/dev/versioning.md](../../dev/versioning.md). ## How you know you need this -Opening a graph whose stamp is below the binary's version fails with: +Opening a graph whose stamp is below the binary's version fails with a +message that **names the release line that wrote it** and the exact commands — +so you can fetch the right old binary without guessing: ``` -__manifest is stamped at internal schema vN, but this omnigraph reads only vM. -This graph was created by an older omnigraph release; rebuild it: run `omnigraph -export` with the older omnigraph binary that created it, then `omnigraph init` + -`omnigraph load` with this one. (Data, vectors, and blobs are preserved; commit -history and branches are not.) +__manifest is stamped at internal schema v3, but this omnigraph reads only v4. +This graph was created by omnigraph 0.6.2 to 0.7.2. Rebuild it: with an omnigraph +0.6.2 to 0.7.2 binary run `omnigraph export > graph.jsonl`, then with this +binary run `omnigraph init --schema ` and `omnigraph load +--mode overwrite --data graph.jsonl `. (Data, vectors, and blobs are +preserved; commit history and branches are not.) See docs/user/operations/upgrade.md. ``` +### Which old binary do I need? + +The on-disk stamp maps to the release line that wrote it. Export with any binary +from that line (the latest is safest): + +| On-disk stamp | Written by | Export with | +|---|---|---| +| internal schema v1 | omnigraph ≤ 0.3.1 | any 0.3.1-or-earlier binary | +| internal schema v2 | omnigraph 0.4.1–0.6.1 | the latest 0.6.x (e.g. 0.6.1) | +| internal schema v3 | omnigraph 0.6.2–0.7.2 | the latest 0.7.x (e.g. 0.7.2) | +| internal schema v4 | omnigraph 0.8.x and later | — current format; no rebuild needed | + You can also check versions before you hit a refusal: - `omnigraph version` — the binary's served version (the `internal-schema ` line).