Skip to content

Commit 62e9104

Browse files
committed
test(cli): full generative cross-backend walk (embedded vs CLI) via omnigraph-dst
The same seeded DST op stream now drives BOTH the embedded Omnigraph SDK and the omnigraph CLI subprocess (Embedded + Cli Backend impls from omnigraph-dst, binary via CARGO_BIN_EXE_omnigraph), asserting per-step black-box agreement on the Person/Doc slug sets + traversable edge count, plus op-kind and op-success parity. Two lockstep walks (same seed + identical reference Model) stay in sync because both wrap the same engine; any divergence fails before they desync. Completes the cross-backend item the plan deferred at the crate boundary (harness in the engine, binary in the CLI). White-box battery stays embedded-only by construction; the CLI arm gets the black-box oracles. The walk surfaced one genuine cross-backend behavioral difference, allow-listed as a known contract (not a transport bug): embedded repair(force=false) returns Ok and leaves suspicious/unverifiable drift (e.g. RC-1's) in place, while the CLI `repair --confirm` (no --force) exits non-zero refusing to publish it — both no-ops on logical data, so the state oracles still agree. Every other op's success/failure matches exactly, including through RC-1 errors and optimize. cargo test -p omnigraph-cli --test cli_cross_backend_walk -> 1 passed.
1 parent a38516e commit 62e9104

3 files changed

Lines changed: 115 additions & 0 deletions

File tree

Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/omnigraph-cli/Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,5 @@ serde_json = { workspace = true }
3434
tempfile = { workspace = true }
3535
lance = { workspace = true }
3636
lance-index = { workspace = true }
37+
omnigraph-dst = { path = "../omnigraph-dst" }
38+
futures = { workspace = true }
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
//! Cross-backend generative walk (PR-E): the SAME seeded DST op sequence driven
2+
//! against the embedded `Omnigraph` SDK AND the `omnigraph` CLI subprocess must
3+
//! agree, per step, on the black-box state (Person/Doc slug sets + traversable
4+
//! edge count).
5+
//!
6+
//! Honest scope: the CLI wraps the SAME engine, so this is NOT a second engine
7+
//! implementation — it verifies the CLI transport layer (arg parsing, output
8+
//! serialization, `--store` addressing, `--as` actor resolution) faithfully
9+
//! reflects the embedded engine across a realistic generative sequence. The
10+
//! white-box invariant battery stays embedded-only (it needs the real handle);
11+
//! this is the black-box half the CLI arm can satisfy. Complements the
12+
//! single-op `parity_matrix.rs` (CLI-local vs CLI-HTTP) and the fixed-sequence
13+
//! `cli_dst_parity.rs` smoke with a generated, model-driven op stream.
14+
//!
15+
//! Lockstep determinism: both walks use the same seed and an identical reference
16+
//! `Model`, and both wrap the same engine, so the op stream matches step-for-
17+
//! step. Any divergence in op-kind, success, or resulting state fails the assert
18+
//! immediately — before the two RNGs/models could desync.
19+
20+
use std::collections::BTreeSet;
21+
use std::path::PathBuf;
22+
23+
use omnigraph_dst::op::OpKind;
24+
use omnigraph_dst::{step, Backend, BackendError, Cli, Embedded, Model, Rng};
25+
26+
/// All slug values for a node type, normalized to an order-independent set. The
27+
/// slug key is matched by suffix so the same parse works for the embedded
28+
/// `to_rust_json` (`"x.slug"`) and the CLI `--json` shape.
29+
async fn slugs<B: Backend>(b: &B, ty: &str) -> BTreeSet<String> {
30+
let q = format!("query q() {{ match {{ $x: {ty} }} return {{ $x.slug }} }}");
31+
let rows = b.query("main", &q).await.expect("slug query");
32+
rows.iter()
33+
.filter_map(|r| {
34+
r.as_object()?
35+
.iter()
36+
.find(|(k, _)| k.ends_with("slug"))
37+
.and_then(|(_, v)| v.as_str())
38+
.map(|s| s.to_string())
39+
})
40+
.collect()
41+
}
42+
43+
/// The traversable `Knows` edge count (`$a knows $b`) — the black-box edge oracle.
44+
async fn edge_count<B: Backend>(b: &B) -> usize {
45+
let rows = b
46+
.query(
47+
"main",
48+
"query q() { match { $a: Person $a knows $b } return { $a.slug, $b.slug } }",
49+
)
50+
.await
51+
.expect("edge query");
52+
rows.len()
53+
}
54+
55+
#[tokio::test]
56+
async fn embedded_and_cli_agree_on_seeded_walk() {
57+
let bin = PathBuf::from(env!("CARGO_BIN_EXE_omnigraph"));
58+
59+
for seed in 0..2u64 {
60+
let emb_dir = tempfile::tempdir().unwrap();
61+
let cli_dir = tempfile::tempdir().unwrap();
62+
let emb = Embedded::open_clean(emb_dir.path().to_str().unwrap()).await;
63+
let cli = Cli::new(bin.clone(), cli_dir.path().to_str().unwrap().to_string());
64+
cli.init().await.expect("cli init");
65+
66+
let mut emb_rng = Rng::new(seed);
67+
let mut cli_rng = Rng::new(seed);
68+
let mut emb_model = Model::new();
69+
let mut cli_model = Model::new();
70+
71+
for i in 0..18 {
72+
let (ek, eres) = step(&emb, &mut emb_rng, &mut emb_model).await;
73+
let (ck, cres) = step(&cli, &mut cli_rng, &mut cli_model).await;
74+
75+
assert_eq!(ek, ck, "seed={seed} step={i}: op-kind diverged");
76+
// Repair's success FLAG legitimately differs across backends and is
77+
// NOT asserted: embedded `repair(force=false)` returns Ok and simply
78+
// leaves suspicious/unverifiable drift in place (e.g. the drift RC-1
79+
// strands), whereas the CLI `repair --confirm` (no --force) EXITS
80+
// NON-ZERO when it refuses to publish such drift. A known contract
81+
// difference, not a transport bug — repair is a no-op on logical
82+
// data either way, so the STATE assertions below must still match.
83+
// Every other op's success/failure must agree exactly (same engine).
84+
if ek != OpKind::Repair {
85+
assert_eq!(
86+
eres.is_ok(),
87+
cres.is_ok(),
88+
"seed={seed} step={i} op[{ek:?}]: success diverged (emb={:?} cli={:?})",
89+
eres.as_ref().err().map(BackendError::message),
90+
cres.as_ref().err().map(BackendError::message),
91+
);
92+
}
93+
94+
assert_eq!(
95+
slugs(&emb, "Person").await,
96+
slugs(&cli, "Person").await,
97+
"seed={seed} step={i} op[{ek:?}]: Person slug set diverged"
98+
);
99+
assert_eq!(
100+
slugs(&emb, "Doc").await,
101+
slugs(&cli, "Doc").await,
102+
"seed={seed} step={i} op[{ek:?}]: Doc slug set diverged"
103+
);
104+
assert_eq!(
105+
edge_count(&emb).await,
106+
edge_count(&cli).await,
107+
"seed={seed} step={i} op[{ek:?}]: traversable edge count diverged"
108+
);
109+
}
110+
}
111+
}

0 commit comments

Comments
 (0)