diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml
index 5225c12..25b6947 100644
--- a/.github/workflows/rust-ci.yml
+++ b/.github/workflows/rust-ci.yml
@@ -14,4 +14,14 @@ permissions:
jobs:
rust-ci:
- uses: hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml@412a7031577112b31ee287cc6060179d638d6500
+ uses: hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml@d135b05bfc647d0c0fbfedc7e80f37ea50f49236
+
+ # The reusable job tests default features only; this keeps the feature-gated
+ # git ingest compiling (it silently broke once — git2 0.21 API drift).
+ feature-git-integration:
+ name: cargo test --features git-integration
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ - run: cargo test --features git-integration
diff --git a/.gitignore b/.gitignore
index d0e9daa..1b95762 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
# Rust build artifacts
/target/
Cargo.lock
+
+# Local lattice cache written by `reticulate build`
+.git-reticulator/
diff --git a/.machine_readable/6a2/STATE.a2ml b/.machine_readable/6a2/STATE.a2ml
index d0d516a..5346153 100644
--- a/.machine_readable/6a2/STATE.a2ml
+++ b/.machine_readable/6a2/STATE.a2ml
@@ -8,8 +8,8 @@
[metadata]
project = "git-reticulator"
version = "0.1.0"
-last-updated = "2026-06-04"
-status = "active" # tidy-up merged (PR #23); migration direction decided 2026-06-04
+last-updated = "2026-07-07"
+status = "active" # dogfood loop landed 2026-07-07 (FileStore + budgeted query, CLI-wired)
[migration-decision]
date = "2026-06-04"
@@ -34,14 +34,15 @@ maturity = "experimental" # experimental | alpha | beta | production | lts
# ════════════════════════════════════════════════════════════════════════════
[honest-status]
lattice-engine = "REAL (src/lattice/mod.rs): Kosaraju SCC-condensation + partial order + LOD zoom (sound+complete) + meet (LCA). cargo green; 10 property/unit tests."
-ingest = "REAL but filesystem-only (src/ingest.rs, std-only, fail-soft). git2 HISTORY ingest NOT yet wired."
-store = "REAL: LatticeStore trait + InMemoryStore default; VeriSimDB octad backend over HTTP behind --features verisim (src/store.rs). Not yet wired into CLI/REST."
-host-reality = "CLI/REST still thin; the compat affine::{build_lattice,query_lattice} shim is now IO-free (no more println-only stubs)."
+ingest = "REAL: std-only filesystem walk (src/ingest.rs, fail-soft) + git2 HEAD-tree/co-change HISTORY ingest behind --features git-integration, wired into the CLI."
+store = "REAL: LatticeStore trait + InMemoryStore + JSON FileStore (versioned envelope, src/store.rs::file) — the default CLI persistence; VeriSimDB octad backend over HTTP behind --features verisim. FileStore wired into CLI build+query 2026-07-07."
+query = "REAL (src/query.rs, 2026-07-07): keyword resolve (case-insensitive, exact-then-coarse ranking) + token-budgeted context packs (chars/4 estimate, drops counted never silent) + text/JSON rendering. This is the dogfood surface — see docs/DOGFOOD.adoc."
+host-reality = "CLI build→query loop REAL end-to-end (ingest → lattice.json → budgeted context pack). REST api still thin; compat affine::{build_lattice,query_lattice} shim retained IO-free."
embeddings = "NOT WIRED (tch/PyTorch feature off by default)"
affine-core = "DEFERRED (ADR-006 bridge-first): the Rust engine is the reference core; the .affine core lands after the Rust↔AffineScript bridge."
proofs = "ZERO mechanized (no .idr/.v/.ads). P2a/P4/P1b are TESTED not proved — see PROOF-NEEDS.md 'Proof status'. 'lattice' = meet-semilattice + digraph (full join not claimed)."
rust-spark = "Stance documented (docs/decisions/rust-spark-stance.adoc) + spark-theatre-gate.yml added (lenient). Idris2/Zig ABI seam N/A until an FFI surface exists."
-tests = "30 pass (10 new lattice/ingest/store + 20 compat integration/property/api). cargo test exit 0."
+tests = "38+ pass (lattice/ingest/store/query incl. FileStore round-trip + budget-truncation honesty + compat integration/property/api). cargo test exit 0 (2026-07-07)."
[crg]
grade = "C"
@@ -70,7 +71,8 @@ template-debt = [
# Core-language question DECIDED 2026-06-04 (ADR-006): AffineScript-first, bridge-first.
actions = [
"DELIVERABLE 1 (in affinescript repo, runtime/): build the Rust loader + marshalling for compiled AffineScript — the affine-js equivalent for Rust. Acceptance: a Rust test loads a compiled .affine fn, calls it, round-trips a string via a host extern fn. See docs/MIGRATION-PLAN.adoc.",
- "LANDED 2026-06-04: Rust reference lattice core (SCC-condensation + partial order + LOD zoom + meet) + filesystem ingest + verisim store seam + 10 tests; cargo green; Rust/SPARK stance + spark-theatre-gate added. NEXT here: wire CLI/REST to ingest→lattice→store, add criterion benches, git2 history ingest.",
+ "LANDED 2026-06-04: Rust reference lattice core (SCC-condensation + partial order + LOD zoom + meet) + filesystem ingest + verisim store seam + 10 tests; cargo green; Rust/SPARK stance + spark-theatre-gate added.",
+ "LANDED 2026-07-07: dogfood loop — JSON FileStore persistence + token-budgeted query engine (src/query.rs) wired into CLI (build writes .git-reticulator/lattice.json, query loads it). NEXT here (docs/DOGFOOD.adoc order): measure token savings on real agent tasks; tree-sitter definition extraction; lattice freshness stamp (HEAD commit); Claude Code skill front-end; criterion benches.",
"Earn the 'lattice' word: SCC-condensation + partial order in the core, then discharge PROOF-NEEDS.md P1-P2 (Idris2).",
"Tests/benches: replace smoke-over-stubs with property tests (partial-order laws, zoom soundness/completeness) + criterion benches; strict-but-passable CI with an AffineScript compile gate.",
]
diff --git a/README.md b/README.md
index 24a2fde..17aeee7 100644
--- a/README.md
+++ b/README.md
@@ -14,14 +14,15 @@ you can **zoom** into, so an LLM gets the minimal relevant context
instead of the whole tree.
> [!IMPORTANT]
-> **Maturity: experimental / early skeleton.** The Rust host is ~237 LOC
-> of `println!` stubs; the lattice core lives in
-> `src/lattice/affine/*.affine` (AffineScript) which **cannot compile
-> yet** and, as written, calls Rust crates AffineScript cannot bind.
-> `git2`/`postgres`/embeddings are feature-gated **off**. There are **no
-> proofs** — the word "lattice" is not yet earned (it is currently a
-> typed digraph; see
-> PROOF-NEEDS). Read
+> **Maturity: experimental.** The Rust reference engine is real
+> (SCC condensation, partial order, LOD zoom, containment meet) and the
+> build→query dogfood loop works end-to-end: `reticulate build` persists
+> a lattice file, `reticulate query` returns token-budgeted context
+> packs (see `docs/DOGFOOD.adoc`). The AffineScript core in
+> `src/lattice/affine/*.affine` is **aspirational** (deferred behind the
+> ADR-006 bridge); `postgres`/embeddings are feature-gated **off**;
+> mechanized proofs cover abstract order theory only, not yet the Rust
+> graph (see PROOF-NEEDS). Read
> `.machine_readable/6a2/STATE.a2ml` for the honest status before
> relying on anything here.
@@ -62,17 +63,22 @@ hallucinated (EXISTENCE). See `.machine_readable/6a2/NEUROSYM.a2ml` and
# Quickstart
```bash
-just build # cargo build (default features; no git2/db/embeddings)
+cargo build --features git-integration # git-aware ingest (plain `cargo build` = filesystem walk)
# CLI binary is `reticulate` (subcommands: build | query | api):
-./target/debug/reticulate build --repo /path/to/repo --db postgres://localhost/gr
-./target/debug/reticulate query --zoom auth --db postgres://localhost/gr
+./target/debug/reticulate build --repo /path/to/repo
+# → ingests the repo, writes /path/to/repo/.git-reticulator/lattice.json
+
+./target/debug/reticulate query --repo /path/to/repo --zoom auth --level definition --budget-tokens 800
+# → token-budgeted context pack (add --format json for machine consumption)
+
./target/debug/reticulate --help
```
> [!NOTE]
-> these run today but are **stubs** — `build` prints and returns; it
-> does not yet read the repo or write the DB.
+> `build` and `query` are **real** end-to-end (ingest → lattice file →
+> budgeted context pack; see `docs/DOGFOOD.adoc`). The `api` server and
+> the VeriSimDB store (`--features verisim`) remain thin.
# Architecture
diff --git a/docs/DOGFOOD.adoc b/docs/DOGFOOD.adoc
new file mode 100644
index 0000000..a7774ad
--- /dev/null
+++ b/docs/DOGFOOD.adoc
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: CC-BY-SA-4.0
+// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell
+
+= Dogfooding git-reticulator: token-bounded repo context
+:toc:
+:revdate: 2026-07-07
+
+== The point
+
+The owner's primary near-term use case is *reducing LLM token spend* during
+agentic coding sessions: when an agent (Claude Code or similar) starts work on
+a repo, it typically burns thousands of tokens on exploratory `ls`/`grep`/file
+reads just to build a mental map. git-reticulator replaces that exploration
+with one pre-built, queryable structure:
+
+[source,bash]
+----
+# Once per repo (rebuild after large changes; cheap enough to re-run freely):
+reticulate build --repo ~/dev/some-repo
+
+# In-session, instead of grep sweeps:
+reticulate query --repo ~/dev/some-repo --zoom auth --level definition --budget-tokens 800
+reticulate query --repo ~/dev/some-repo --zoom store --level file --format json
+----
+
+`build` ingests the repo (git-aware with `--features git-integration`: HEAD
+tree + commit co-change coupling; plain filesystem walk otherwise) and writes
+`/.git-reticulator/lattice.json`. `query` resolves a keyword against the
+lattice, zooms each match to the requested level-of-detail, and renders a
+context pack that *fits the stated token budget* — anything dropped is counted
+and reported, never silently omitted.
+
+== How this reduces token count, concretely
+
+1. *Map once, query many.* The lattice is built offline (zero LLM tokens).
+ Each query returns only the relevant subtree at the requested granularity.
+2. *Explicit budget.* `--budget-tokens N` caps the pack (chars/4 estimate), so
+ a context injection can never blow out a prompt.
+3. *Agent integration.* The intended consumption path is a Claude Code skill /
+ `CLAUDE.md` instruction of the form: "before grep-exploring, run
+ `reticulate query --zoom ` and only read the files it names."
+ That converts N speculative file reads into one bounded text block.
+
+== Honest status of the claim
+
+The *mechanism* works end-to-end as of this document's date (build → file →
+query, tested). The *savings* are not yet measured. Before wiring this into
+every session, run the experiment:
+
+* Pick 3 real tasks on a mid-size repo.
+* Run each twice: once with normal agent exploration, once with a
+ "query-the-lattice-first" instruction.
+* Compare input-token counts and whether the agent found the right files.
+
+If the packs lose to plain exploration, the fix is better ingestion (real
+symbol extraction via tree-sitter rather than the current line-prefix
+heuristic), not more infrastructure.
+
+== What is deliberately NOT needed for this loop
+
+* *VeriSimDB / postgres* — the JSON file store is enough for one repo.
+* *Embeddings* — symbolic keyword resolution first; neural similarity later.
+* *AffineScript core* — the Rust reference engine is the core until the
+ bridge lands (ADR-006, tracked in the `affinescript` repo).
+* *Proofs* — PROOF-NEEDS.md governs the "lattice" claim, not the dogfood loop.
+
+== Path to production-ready (proposed order)
+
+1. *Dogfood loop* (this document) — DONE: persistence + budgeted query.
+2. *Measurement* — the experiment above; publish numbers in this file.
+3. *Ingestion quality* — tree-sitter (or per-language) definition extraction;
+ `calls`/`imports` edges, not just containment + co-change.
+4. *Freshness* — record the HEAD commit in the lattice file; `query` warns
+ when the lattice is stale relative to the repo.
+5. *Agent surface* — a Claude Code skill (estate-wide) that fronts `reticulate
+ query`; optionally the REST API for non-CLI consumers.
+6. *CRG B* — lint/fmt/doc-coverage targets per READINESS.md.
+7. *The neuro-symbolic stack* — embeddings, VeriSimDB, proof-carrying
+ retrieval — only after steps 1–5 prove the symbolic half pays for itself.
diff --git a/src/cli/main.rs b/src/cli/main.rs
index b53cf9d..d2aed86 100644
--- a/src/cli/main.rs
+++ b/src/cli/main.rs
@@ -1,8 +1,13 @@
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) Jonathan D.A. Jewell
-use clap::{Parser, Subcommand};
-use git_reticulator::lattice::affine;
+use clap::{Parser, Subcommand, ValueEnum};
+use git_reticulator::lattice::SemanticLevel;
+use git_reticulator::store::file::FileStore;
use git_reticulator::store::LatticeStore;
+use std::path::PathBuf;
+
+/// Default lattice file location relative to the ingested repo.
+const DEFAULT_LATTICE_REL: &str = ".git-reticulator/lattice.json";
#[derive(Parser)]
#[command(name = "reticulate")]
@@ -12,25 +17,66 @@ struct Cli {
command: Commands,
}
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum LevelArg {
+ Module,
+ File,
+ Definition,
+ Block,
+}
+
+impl From for SemanticLevel {
+ fn from(l: LevelArg) -> Self {
+ match l {
+ LevelArg::Module => SemanticLevel::Module,
+ LevelArg::File => SemanticLevel::File,
+ LevelArg::Definition => SemanticLevel::Definition,
+ LevelArg::Block => SemanticLevel::Block,
+ }
+ }
+}
+
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum FormatArg {
+ Text,
+ Json,
+}
+
#[derive(Subcommand)]
enum Commands {
- /// Build a semantic lattice from a git repo
+ /// Build a semantic lattice from a repo and persist it to a local lattice
+ /// file (and optionally to VeriSimDB with --features verisim)
Build {
- /// Path to the git repository
- #[arg(short, long)]
+ /// Path to the repository to ingest
+ #[arg(short, long, default_value = ".")]
repo: String,
- /// PostgreSQL database URI
+ /// Output lattice file (default: /.git-reticulator/lattice.json)
#[arg(short, long)]
- db: String,
+ out: Option,
+ /// VeriSimDB base URL (http/https; requires --features verisim)
+ #[arg(short, long)]
+ db: Option,
},
- /// Query the lattice with a zoom level to minimize token cost
+ /// Query a built lattice for a token-budgeted context pack
Query {
- /// Semantic node or keyword to zoom into
+ /// Keyword to resolve (case-insensitive substring over node names)
#[arg(short, long)]
zoom: String,
- /// PostgreSQL database URI
+ /// Lattice file to query (default: /.git-reticulator/lattice.json)
#[arg(short, long)]
- db: String,
+ lattice: Option,
+ /// Repository the lattice was built from (locates the default lattice file)
+ #[arg(short, long, default_value = ".")]
+ repo: String,
+ /// Level-of-detail to zoom matches to
+ #[arg(long, value_enum, default_value_t = LevelArg::Definition)]
+ level: LevelArg,
+ /// Output format
+ #[arg(short, long, value_enum, default_value_t = FormatArg::Text)]
+ format: FormatArg,
+ /// Token budget for the rendered context pack (chars/4 estimate)
+ #[arg(short, long, default_value_t = 2000)]
+ budget_tokens: usize,
},
/// Start the REST API server for LLM integration
Api {
@@ -56,13 +102,17 @@ fn reticulate_ingest(repo: &str) -> git_reticulator::lattice::Lattice {
git_reticulator::ingest::from_path(repo)
}
+fn default_lattice_path(repo: &str) -> PathBuf {
+ PathBuf::from(repo).join(DEFAULT_LATTICE_REL)
+}
+
#[tokio::main]
async fn main() {
let cli = Cli::parse();
env_logger::init();
match &cli.command {
- Commands::Build { repo, db } => {
+ Commands::Build { repo, out, db } => {
println!("🚀 Reticulating {repo} ...");
let lattice = reticulate_ingest(repo);
let cond = lattice.condense();
@@ -74,34 +124,66 @@ async fn main() {
cond.is_acyclic()
);
+ let out_path = out.clone().unwrap_or_else(|| default_lattice_path(repo));
+ let mut store = FileStore::new(&out_path);
+ match store.persist(&lattice) {
+ Ok(n) => println!("📦 persisted {n} nodes to {}", out_path.display()),
+ Err(e) => {
+ eprintln!("❌ cannot write {}: {e}", out_path.display());
+ std::process::exit(1);
+ }
+ }
+
#[cfg(feature = "verisim")]
- let to_verisim = if db.starts_with("http://") || db.starts_with("https://") {
- let store = git_reticulator::store::verisim::VerisimStore::new(db.clone());
- match store.persist(&lattice).await {
- Ok(n) => println!("📦 persisted {n} octads to VeriSimDB ({db})"),
- Err(e) => eprintln!("⚠️ verisim persist failed: {e}"),
+ if let Some(db) = db {
+ if db.starts_with("http://") || db.starts_with("https://") {
+ let store = git_reticulator::store::verisim::VerisimStore::new(db.clone());
+ match store.persist(&lattice).await {
+ Ok(n) => println!("📦 persisted {n} octads to VeriSimDB ({db})"),
+ Err(e) => eprintln!("⚠️ verisim persist failed: {e}"),
+ }
}
- true
- } else {
- false
- };
+ }
#[cfg(not(feature = "verisim"))]
- let to_verisim = false;
-
- if !to_verisim {
- let mut store = git_reticulator::store::InMemoryStore::new();
- let n = match store.persist(&lattice) {
- Ok(n) => n,
- // InMemoryStore is Infallible — this arm is unreachable.
- Err(never) => match never {},
- };
- println!("📦 persisted {n} nodes to the in-memory store (target: {db})");
+ if let Some(db) = db {
+ eprintln!("⚠️ --db {db} ignored: rebuild with --features verisim");
}
+
println!("✅ done.");
}
- Commands::Query { zoom, db } => {
- println!("🔍 Querying lattice for context: {}", zoom);
- affine::query_lattice(zoom, db);
+ Commands::Query {
+ zoom,
+ lattice,
+ repo,
+ level,
+ format,
+ budget_tokens,
+ } => {
+ let path = lattice
+ .clone()
+ .unwrap_or_else(|| default_lattice_path(repo));
+ let lat = match FileStore::load(&path) {
+ Ok(lat) => lat,
+ Err(e) => {
+ eprintln!(
+ "❌ {e}\n no usable lattice at {} — run `reticulate build --repo {repo}` first",
+ path.display()
+ );
+ std::process::exit(1);
+ }
+ };
+ let result =
+ git_reticulator::query::context_pack(&lat, zoom, (*level).into(), *budget_tokens);
+ match format {
+ FormatArg::Text => print!("{}", git_reticulator::query::render_text(&result)),
+ FormatArg::Json => match serde_json::to_string_pretty(&result) {
+ Ok(json) => println!("{json}"),
+ Err(e) => {
+ eprintln!("❌ cannot serialize result: {e}");
+ std::process::exit(1);
+ }
+ },
+ }
}
Commands::Api { db } => {
println!("🌐 Starting Git-Reticulator API on http://localhost:8080");
diff --git a/src/ingest.rs b/src/ingest.rs
index e5fbd7b..da52759 100644
--- a/src/ingest.rs
+++ b/src/ingest.rs
@@ -70,7 +70,12 @@ fn walk(builder: &mut LatticeBuilder, dir: &Path, parent: NodeId, depth: usize)
if let Ok(content) = fs::read_to_string(&path) {
let file_disp = path.to_string_lossy().to_string();
for def in extract_definitions(&content) {
- builder.add_keyword(def, file_disp.clone(), SemanticLevel::Definition, Some(file_id));
+ builder.add_keyword(
+ def,
+ file_disp.clone(),
+ SemanticLevel::Definition,
+ Some(file_id),
+ );
}
}
}
@@ -81,8 +86,17 @@ fn walk(builder: &mut LatticeBuilder, dir: &Path, parent: NodeId, depth: usize)
/// identifier following a common definition keyword at the start of a line.
fn extract_definitions(content: &str) -> Vec {
const KEYWORDS: [&str; 11] = [
- "pub fn ", "fn ", "def ", "class ", "struct ", "enum ", "trait ", "type ", "module ",
- "interface ", "func ",
+ "pub fn ",
+ "fn ",
+ "def ",
+ "class ",
+ "struct ",
+ "enum ",
+ "trait ",
+ "type ",
+ "module ",
+ "interface ",
+ "func ",
];
let mut defs = Vec::new();
for line in content.lines() {
@@ -125,7 +139,8 @@ mod tests {
#[test]
fn extracts_definitions_from_source_text() {
- let defs = extract_definitions("pub fn login() {}\nstruct Session;\n// comment\nfn helper() {}");
+ let defs =
+ extract_definitions("pub fn login() {}\nstruct Session;\n// comment\nfn helper() {}");
assert!(defs.contains(&"login".to_string()));
assert!(defs.contains(&"Session".to_string()));
assert!(defs.contains(&"helper".to_string()));
@@ -173,8 +188,12 @@ mod git_history {
.and_then(|s| s.to_str())
.map(String::from)
.unwrap_or_else(|| repo_path.to_string());
- let root_id =
- builder.add_keyword(root_name, repo_path.to_string(), SemanticLevel::Module, None);
+ let root_id = builder.add_keyword(
+ root_name,
+ repo_path.to_string(),
+ SemanticLevel::Module,
+ None,
+ );
// 1. Structure from the HEAD tree. `dir_ids` is keyed by the
// trailing-slash directory path git2 hands the walk callback ("" is
@@ -188,8 +207,8 @@ mod git_history {
let head = repo.head()?.peel_to_tree()?;
head.walk(git2::TreeWalkMode::PreOrder, |dir, entry| {
let name = match entry.name() {
- Some(n) => n.to_string(),
- None => return git2::TreeWalkResult::Ok, // non-UTF-8 path: skip
+ Ok(n) => n.to_string(),
+ Err(_) => return git2::TreeWalkResult::Ok, // non-UTF-8 path: skip
};
let parent = dir_ids.get(dir).copied().unwrap_or(root_id);
match entry.kind() {
@@ -220,8 +239,16 @@ mod git_history {
for (fid, oid, path) in &blobs {
if let Ok(blob) = repo.find_blob(*oid) {
if let Ok(text) = std::str::from_utf8(blob.content()) {
- for def in extract_definitions(text).into_iter().take(MAX_DEFS_PER_FILE) {
- builder.add_keyword(def, path.clone(), SemanticLevel::Definition, Some(*fid));
+ for def in extract_definitions(text)
+ .into_iter()
+ .take(MAX_DEFS_PER_FILE)
+ {
+ builder.add_keyword(
+ def,
+ path.clone(),
+ SemanticLevel::Definition,
+ Some(*fid),
+ );
}
}
}
@@ -244,11 +271,11 @@ mod git_history {
Err(_) => continue,
};
let parent_tree = commit.parent(0).ok().and_then(|p| p.tree().ok());
- let diff =
- match repo.diff_tree_to_tree(parent_tree.as_ref(), Some(&tree), None) {
- Ok(d) => d,
- Err(_) => continue,
- };
+ let diff = match repo.diff_tree_to_tree(parent_tree.as_ref(), Some(&tree), None)
+ {
+ Ok(d) => d,
+ Err(_) => continue,
+ };
let mut changed: Vec = Vec::new();
for delta in diff.deltas() {
let path = delta.new_file().path().or_else(|| delta.old_file().path());
@@ -291,7 +318,10 @@ mod git_history_tests {
let Ok(lat) = from_git(".") else {
panic!("the package working directory should be a git repository");
};
- assert!(lat.len() > 1, "the HEAD tree should yield more than the root module");
+ assert!(
+ lat.len() > 1,
+ "the HEAD tree should yield more than the root module"
+ );
assert!(
lat.nodes().iter().any(|k| k.name == "Cargo.toml"),
"the tracked Cargo.toml should appear as a File node"
diff --git a/src/lattice/mod.rs b/src/lattice/mod.rs
index 86d25f9..767bb45 100644
--- a/src/lattice/mod.rs
+++ b/src/lattice/mod.rs
@@ -13,6 +13,7 @@
// replace it without touching the host (ADR-001, AffineScript-first target).
// All IO (git ingestion, verisim persistence) lives outside this module.
+use serde::{Deserialize, Serialize};
use std::collections::{BTreeSet, HashMap, VecDeque};
/// Index of a node into [`Lattice::nodes`]. Stable for the lattice's lifetime.
@@ -20,7 +21,8 @@ pub type NodeId = usize;
/// Level-of-detail tier. The containment hierarchy (`parent`) runs
/// Module ⊃ File ⊃ Definition ⊃ Block.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
pub enum SemanticLevel {
Module,
File,
@@ -51,19 +53,21 @@ impl SemanticLevel {
/// A semantic keyword node: simultaneously a lattice element (order position via
/// `parent`/edges) and a neural element (`embedding`) — the neuro-symbolic seam.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Keyword {
pub id: NodeId,
pub name: String,
pub file: String,
pub level: SemanticLevel,
pub parent: Option,
+ #[serde(skip_serializing_if = "Option::is_none", default)]
pub embedding: Option>,
+ #[serde(skip_serializing_if = "Option::is_none", default)]
pub cluster: Option,
}
/// A typed, weighted relationship (calls / contains / depends_on / …).
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Relationship {
pub source: NodeId,
pub target: NodeId,
@@ -133,7 +137,7 @@ impl Condensation {
/// A semantic lattice: typed keyword nodes + weighted relationships, with the
/// algebra that earns the name (condensation, partial order, meet, LOD zoom).
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Lattice {
nodes: Vec,
edges: Vec,
@@ -393,7 +397,12 @@ pub mod affine {
/// reports an engine summary. Never touches the filesystem or network.
pub fn build_lattice(repo: &str, db: &str) {
let mut builder = LatticeBuilder::new();
- builder.add_keyword(repo.to_string(), repo.to_string(), SemanticLevel::Module, None);
+ builder.add_keyword(
+ repo.to_string(),
+ repo.to_string(),
+ SemanticLevel::Module,
+ None,
+ );
let lattice = builder.build();
let cond = lattice.condense();
println!(
@@ -406,7 +415,9 @@ pub mod affine {
/// Compat entry point for a zoom request. IO-free.
pub fn query_lattice(zoom: &str, db: &str) {
- println!("zoom target '{zoom}' [source: {db}] — run `reticulate build` to populate a lattice");
+ println!(
+ "zoom target '{zoom}' [source: {db}] — run `reticulate build` to populate a lattice"
+ );
}
}
@@ -419,11 +430,36 @@ mod tests {
fn fixture() -> Lattice {
let mut b = LatticeBuilder::new();
let m = b.add_keyword("root".into(), "/".into(), SemanticLevel::Module, None);
- let f1 = b.add_keyword("auth.rs".into(), "/auth.rs".into(), SemanticLevel::File, Some(m));
- let f2 = b.add_keyword("db.rs".into(), "/db.rs".into(), SemanticLevel::File, Some(m));
- let d1 = b.add_keyword("login".into(), "/auth.rs".into(), SemanticLevel::Definition, Some(f1));
- let d2 = b.add_keyword("session".into(), "/auth.rs".into(), SemanticLevel::Definition, Some(f1));
- let d3 = b.add_keyword("connect".into(), "/db.rs".into(), SemanticLevel::Definition, Some(f2));
+ let f1 = b.add_keyword(
+ "auth.rs".into(),
+ "/auth.rs".into(),
+ SemanticLevel::File,
+ Some(m),
+ );
+ let f2 = b.add_keyword(
+ "db.rs".into(),
+ "/db.rs".into(),
+ SemanticLevel::File,
+ Some(m),
+ );
+ let d1 = b.add_keyword(
+ "login".into(),
+ "/auth.rs".into(),
+ SemanticLevel::Definition,
+ Some(f1),
+ );
+ let d2 = b.add_keyword(
+ "session".into(),
+ "/auth.rs".into(),
+ SemanticLevel::Definition,
+ Some(f1),
+ );
+ let d3 = b.add_keyword(
+ "connect".into(),
+ "/db.rs".into(),
+ SemanticLevel::Definition,
+ Some(f2),
+ );
// login -> session -> connect -> login (a cycle, to exercise SCC)
b.add_relationship(d1, d2, 1.0, "calls".into());
b.add_relationship(d2, d3, 1.0, "calls".into());
diff --git a/src/lib.rs b/src/lib.rs
index 860fefa..0618bf0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,14 +12,18 @@
// * `lattice` — the pure, dependency-free engine (SCC condensation, partial
// order, LOD zoom, containment meet). Reference core today; designed to be
// swapped for an AffineScript→Wasm core later (ADR-001) without host churn.
-// * `ingest` — repository → lattice (std-only filesystem walk).
-// * `store` — persistence seam; VeriSimDB octad backend behind `--features verisim`.
+// * `ingest` — repository → lattice (std-only filesystem walk; git-aware
+// HEAD-tree + co-change ingest behind `--features git-integration`).
+// * `store` — persistence seam; JSON `FileStore` for the standalone
+// build→query loop, VeriSimDB octad backend behind `--features verisim`.
+// * `query` — token-budgeted context packs over a built lattice.
// * `api` — actix-web REST surface.
#![forbid(unsafe_code)]
-pub mod lattice;
pub mod ingest;
+pub mod lattice;
+pub mod query;
pub mod store;
pub mod api {
diff --git a/src/query.rs b/src/query.rs
new file mode 100644
index 0000000..d40349d
--- /dev/null
+++ b/src/query.rs
@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2026 Jonathan D.A. Jewell
+//
+// src/query.rs
+//
+// Token-budgeted context packs over a built lattice. This is the consumer
+// surface of the dogfood loop: resolve a keyword to lattice nodes, zoom each
+// match to the requested level-of-detail, and render the result within an
+// explicit token budget — reporting exactly what was dropped, never silently
+// truncating (a truncated pack that looks complete would defeat the point).
+//
+// Token accounting is the standard chars/4 heuristic. It only has to be
+// honest enough to keep a pack near the budget the caller asked for; the
+// caller's tokenizer is the ground truth.
+
+use crate::lattice::{Lattice, NodeId, SemanticLevel};
+use serde::Serialize;
+
+/// Approximate tokens for a rendered string (chars/4, minimum 1 per line).
+fn estimate_tokens(s: &str) -> usize {
+ (s.chars().count() / 4).max(1)
+}
+
+/// A node as it appears in a context pack.
+#[derive(Clone, Debug, Serialize)]
+pub struct NodeInfo {
+ pub id: NodeId,
+ pub name: String,
+ pub file: String,
+ pub level: String,
+}
+
+impl NodeInfo {
+ fn from_lattice(lattice: &Lattice, id: NodeId) -> Option {
+ lattice.node(id).map(|k| NodeInfo {
+ id: k.id,
+ name: k.name.clone(),
+ file: k.file.clone(),
+ level: k.level.as_str().to_string(),
+ })
+ }
+}
+
+/// One matched node plus its zoomed context.
+#[derive(Clone, Debug, Serialize)]
+pub struct MatchPack {
+ pub node: NodeInfo,
+ /// Containment path from the root to the matched node (names, coarse→fine).
+ pub path: Vec,
+ /// Descendants of the match at the requested level, in lattice order.
+ pub descendants: Vec,
+ /// Descendants that existed but were dropped to stay within budget.
+ pub descendants_dropped: usize,
+}
+
+/// The full result of a query: every match that fit the budget, plus an
+/// honest account of what did not.
+#[derive(Clone, Debug, Serialize)]
+pub struct QueryResult {
+ pub pattern: String,
+ pub level: String,
+ pub matches: Vec,
+ /// Matched nodes dropped entirely because the budget was exhausted.
+ pub matches_dropped: usize,
+ /// Estimated tokens of the text rendering of this result.
+ pub estimated_tokens: usize,
+ pub budget_tokens: usize,
+}
+
+/// Case-insensitive substring match on node names. Exact (case-insensitive)
+/// matches sort first, then coarser levels before finer, then insertion order —
+/// so a module named `auth` beats a definition named `authorize_retry`.
+pub fn resolve(lattice: &Lattice, pattern: &str) -> Vec {
+ let needle = pattern.to_lowercase();
+ let mut hits: Vec = lattice
+ .nodes()
+ .iter()
+ .filter(|k| k.name.to_lowercase().contains(&needle))
+ .map(|k| k.id)
+ .collect();
+ hits.sort_by_key(|&id| {
+ let k = &lattice.nodes()[id];
+ let exact = k.name.to_lowercase() != needle; // false (exact) sorts first
+ (exact, k.level.rank(), id)
+ });
+ hits
+}
+
+/// Containment path from the root to `id` (names, coarse→fine). Guards against
+/// malformed parent cycles the same way the lattice core does.
+fn containment_path(lattice: &Lattice, id: NodeId) -> Vec {
+ let mut path = Vec::new();
+ let mut cur = Some(id);
+ let mut guard = 0;
+ while let Some(x) = cur {
+ if guard > lattice.len() {
+ break;
+ }
+ match lattice.node(x) {
+ Some(k) => {
+ path.push(k.name.clone());
+ cur = k.parent;
+ }
+ None => break,
+ }
+ guard += 1;
+ }
+ path.reverse();
+ path
+}
+
+/// Build a token-budgeted context pack: resolve `pattern`, zoom each match to
+/// `level`, and include as much as fits in `budget_tokens` (estimated on the
+/// text rendering). Whatever is dropped is counted, never hidden.
+pub fn context_pack(
+ lattice: &Lattice,
+ pattern: &str,
+ level: SemanticLevel,
+ budget_tokens: usize,
+) -> QueryResult {
+ let hits = resolve(lattice, pattern);
+ let mut result = QueryResult {
+ pattern: pattern.to_string(),
+ level: level.as_str().to_string(),
+ matches: Vec::new(),
+ matches_dropped: 0,
+ estimated_tokens: 0,
+ budget_tokens,
+ };
+
+ let mut spent = 0usize;
+ for (i, &id) in hits.iter().enumerate() {
+ let node = match NodeInfo::from_lattice(lattice, id) {
+ Some(n) => n,
+ None => continue,
+ };
+ let path = containment_path(lattice, id);
+ let header_cost = estimate_tokens(&format!(
+ "## {} ({}) — {}\npath: {}\n",
+ node.name,
+ node.level,
+ node.file,
+ path.join(" > ")
+ ));
+ if spent + header_cost > budget_tokens && !result.matches.is_empty() {
+ // No room for even this match's header: drop it and the rest.
+ result.matches_dropped = hits.len() - i;
+ break;
+ }
+ spent += header_cost;
+
+ let zoomed = lattice.zoom(id, level);
+ let mut descendants = Vec::new();
+ let mut dropped = 0usize;
+ for &d in &zoomed {
+ let info = match NodeInfo::from_lattice(lattice, d) {
+ Some(n) => n,
+ None => continue,
+ };
+ let line_cost =
+ estimate_tokens(&format!("- {} [{}] {}\n", info.name, info.level, info.file));
+ if spent + line_cost > budget_tokens {
+ dropped = zoomed.len() - descendants.len();
+ break;
+ }
+ spent += line_cost;
+ descendants.push(info);
+ }
+
+ result.matches.push(MatchPack {
+ node,
+ path,
+ descendants,
+ descendants_dropped: dropped,
+ });
+
+ if dropped > 0 {
+ // Budget exhausted mid-match: everything after this match is dropped.
+ result.matches_dropped = hits.len() - i - 1;
+ break;
+ }
+ }
+
+ result.estimated_tokens = spent;
+ result
+}
+
+/// Render a [`QueryResult`] as compact, LLM-ready text.
+pub fn render_text(result: &QueryResult) -> String {
+ let mut out = String::new();
+ if result.matches.is_empty() && result.matches_dropped == 0 {
+ out.push_str(&format!("no nodes match '{}'\n", result.pattern));
+ return out;
+ }
+ for m in &result.matches {
+ out.push_str(&format!(
+ "## {} ({}) — {}\npath: {}\n",
+ m.node.name,
+ m.node.level,
+ m.node.file,
+ m.path.join(" > ")
+ ));
+ for d in &m.descendants {
+ out.push_str(&format!("- {} [{}] {}\n", d.name, d.level, d.file));
+ }
+ if m.descendants_dropped > 0 {
+ out.push_str(&format!(
+ "… {} more {} node(s) omitted (budget)\n",
+ m.descendants_dropped, result.level
+ ));
+ }
+ }
+ if result.matches_dropped > 0 {
+ out.push_str(&format!(
+ "… {} more match(es) omitted (budget {} tokens)\n",
+ result.matches_dropped, result.budget_tokens
+ ));
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::lattice::LatticeBuilder;
+
+ fn fixture() -> Lattice {
+ let mut b = LatticeBuilder::new();
+ let m = b.add_keyword("root".into(), "/".into(), SemanticLevel::Module, None);
+ let auth = b.add_keyword(
+ "auth".into(),
+ "/auth".into(),
+ SemanticLevel::Module,
+ Some(m),
+ );
+ let f1 = b.add_keyword(
+ "auth.rs".into(),
+ "/auth/auth.rs".into(),
+ SemanticLevel::File,
+ Some(auth),
+ );
+ b.add_keyword(
+ "login".into(),
+ "/auth/auth.rs".into(),
+ SemanticLevel::Definition,
+ Some(f1),
+ );
+ b.add_keyword(
+ "authorize_retry".into(),
+ "/auth/auth.rs".into(),
+ SemanticLevel::Definition,
+ Some(f1),
+ );
+ let f2 = b.add_keyword(
+ "db.rs".into(),
+ "/db.rs".into(),
+ SemanticLevel::File,
+ Some(m),
+ );
+ b.add_keyword(
+ "connect".into(),
+ "/db.rs".into(),
+ SemanticLevel::Definition,
+ Some(f2),
+ );
+ b.build()
+ }
+
+ #[test]
+ fn resolve_prefers_exact_then_coarse() {
+ let lat = fixture();
+ let hits = resolve(&lat, "auth");
+ // exact module 'auth' (id 1) first, then substring hits
+ assert_eq!(hits[0], 1);
+ assert!(hits.contains(&2)); // auth.rs
+ assert!(hits.contains(&4)); // authorize_retry
+ }
+
+ #[test]
+ fn resolve_is_case_insensitive_and_misses_cleanly() {
+ let lat = fixture();
+ assert!(!resolve(&lat, "LOGIN").is_empty());
+ assert!(resolve(&lat, "zebra").is_empty());
+ }
+
+ #[test]
+ fn context_pack_zooms_matches_to_level() {
+ let lat = fixture();
+ let result = context_pack(&lat, "auth", SemanticLevel::Definition, 10_000);
+ assert_eq!(result.matches_dropped, 0);
+ let first = &result.matches[0];
+ assert_eq!(first.node.name, "auth");
+ assert_eq!(first.path, vec!["root", "auth"]);
+ let names: Vec<_> = first.descendants.iter().map(|d| d.name.as_str()).collect();
+ assert!(names.contains(&"login"));
+ assert!(names.contains(&"authorize_retry"));
+ assert!(!names.contains(&"connect")); // soundness: other subtree excluded
+ }
+
+ #[test]
+ fn budget_truncates_and_reports_drops() {
+ let lat = fixture();
+ let full = context_pack(&lat, "auth", SemanticLevel::Definition, 10_000);
+ let full_tokens = full.estimated_tokens;
+ assert!(full_tokens > 12);
+
+ let tight = context_pack(&lat, "auth", SemanticLevel::Definition, 12);
+ assert!(
+ tight.estimated_tokens <= 12,
+ "spent {} > budget",
+ tight.estimated_tokens
+ );
+ let dropped_somewhere =
+ tight.matches_dropped > 0 || tight.matches.iter().any(|m| m.descendants_dropped > 0);
+ assert!(dropped_somewhere, "a tight budget must report drops");
+ // ... and the drops are visible in the rendering, not silent.
+ let text = render_text(&tight);
+ assert!(text.contains("omitted"));
+ }
+
+ #[test]
+ fn render_text_reports_no_match() {
+ let lat = fixture();
+ let result = context_pack(&lat, "zebra", SemanticLevel::Definition, 100);
+ assert!(render_text(&result).contains("no nodes match"));
+ }
+}
diff --git a/src/store.rs b/src/store.rs
index 72e39bf..ac879b6 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -48,6 +48,105 @@ impl LatticeStore for InMemoryStore {
}
}
+/// File-backed store: a versioned JSON envelope on local disk. This is the
+/// default persistence for the dogfood loop (`reticulate build` → file →
+/// `reticulate query`) — no database required. VeriSimDB remains the intended
+/// database of record for the full neuro-symbolic stack; this store exists so
+/// the build→query loop works standalone today.
+pub mod file {
+ use crate::lattice::Lattice;
+ use serde::{Deserialize, Serialize};
+ use std::fs;
+ use std::path::{Path, PathBuf};
+
+ /// Bumped whenever the on-disk shape changes incompatibly.
+ pub const FORMAT_VERSION: u32 = 1;
+ const FORMAT_NAME: &str = "git-reticulator/lattice";
+
+ #[derive(Serialize, Deserialize)]
+ struct Envelope {
+ format: String,
+ version: u32,
+ lattice: Lattice,
+ }
+
+ /// Errors from loading a lattice file.
+ #[derive(Debug)]
+ pub enum LoadError {
+ Io(std::io::Error),
+ Parse(serde_json::Error),
+ /// The file parsed but is not a lattice file, or its version is unsupported.
+ Format(String),
+ }
+
+ impl std::fmt::Display for LoadError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ LoadError::Io(e) => write!(f, "cannot read lattice file: {e}"),
+ LoadError::Parse(e) => write!(f, "cannot parse lattice file: {e}"),
+ LoadError::Format(msg) => write!(f, "unsupported lattice file: {msg}"),
+ }
+ }
+ }
+
+ /// Persist a lattice as JSON at `path`; `LatticeStore::persist` creates
+ /// parent directories as needed and overwrites any existing file.
+ #[derive(Debug)]
+ pub struct FileStore {
+ path: PathBuf,
+ }
+
+ impl FileStore {
+ pub fn new(path: impl Into) -> Self {
+ Self { path: path.into() }
+ }
+
+ pub fn path(&self) -> &Path {
+ &self.path
+ }
+
+ /// Load a lattice previously written by [`FileStore`].
+ pub fn load(path: &Path) -> Result {
+ let text = fs::read_to_string(path).map_err(LoadError::Io)?;
+ let envelope: Envelope = serde_json::from_str(&text).map_err(LoadError::Parse)?;
+ if envelope.format != FORMAT_NAME {
+ return Err(LoadError::Format(format!(
+ "format is '{}', expected '{FORMAT_NAME}'",
+ envelope.format
+ )));
+ }
+ if envelope.version != FORMAT_VERSION {
+ return Err(LoadError::Format(format!(
+ "version {} not supported (this build reads version {FORMAT_VERSION})",
+ envelope.version
+ )));
+ }
+ Ok(envelope.lattice)
+ }
+ }
+
+ impl super::LatticeStore for FileStore {
+ type Error = std::io::Error;
+
+ fn persist(&mut self, lattice: &Lattice) -> Result {
+ if let Some(parent) = self.path.parent() {
+ if !parent.as_os_str().is_empty() {
+ fs::create_dir_all(parent)?;
+ }
+ }
+ let envelope = Envelope {
+ format: FORMAT_NAME.to_string(),
+ version: FORMAT_VERSION,
+ lattice: lattice.clone(),
+ };
+ let json = serde_json::to_string(&envelope)
+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+ fs::write(&self.path, json)?;
+ Ok(lattice.len())
+ }
+ }
+}
+
/// VeriSimDB octad-store backend (feature `verisim`). Talks to `verisim-api`
/// over HTTP; see the module docs for the modality mapping.
#[cfg(feature = "verisim")]
@@ -136,4 +235,56 @@ mod tests {
assert_eq!(n, 2);
assert_eq!(store.stored(), 2);
}
+
+ #[test]
+ fn file_store_round_trips_a_lattice() {
+ let mut b = LatticeBuilder::new();
+ let m = b.add_keyword("root".into(), "/".into(), SemanticLevel::Module, None);
+ let f = b.add_keyword("a.rs".into(), "/a.rs".into(), SemanticLevel::File, Some(m));
+ b.add_keyword(
+ "login".into(),
+ "/a.rs".into(),
+ SemanticLevel::Definition,
+ Some(f),
+ );
+ b.add_relationship(f, m, 1.0, "contains".into());
+ let lattice = b.build();
+
+ let path = std::env::temp_dir().join(format!(
+ "git-reticulator-roundtrip-{}.json",
+ std::process::id()
+ ));
+ let mut store = file::FileStore::new(&path);
+ let n = store.persist(&lattice).unwrap();
+ assert_eq!(n, 3);
+
+ let loaded = file::FileStore::load(&path).unwrap();
+ assert_eq!(loaded.len(), lattice.len());
+ assert_eq!(loaded.edges().len(), lattice.edges().len());
+ assert_eq!(loaded.node(2).unwrap().name, "login");
+ assert_eq!(loaded.node(2).unwrap().parent, Some(1));
+ let _ = std::fs::remove_file(&path);
+ }
+
+ #[test]
+ fn file_store_load_rejects_garbage_and_missing() {
+ assert!(matches!(
+ file::FileStore::load(std::path::Path::new("/no/such/lattice.json")),
+ Err(file::LoadError::Io(_))
+ ));
+ let path = std::env::temp_dir().join(format!(
+ "git-reticulator-garbage-{}.json",
+ std::process::id()
+ ));
+ std::fs::write(
+ &path,
+ "{\"format\":\"something-else\",\"version\":1,\"lattice\":{\"nodes\":[],\"edges\":[]}}",
+ )
+ .unwrap();
+ assert!(matches!(
+ file::FileStore::load(&path),
+ Err(file::LoadError::Format(_))
+ ));
+ let _ = std::fs::remove_file(&path);
+ }
}
diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs
index 1810d76..2d98be6 100644
--- a/tests/integration_tests.rs
+++ b/tests/integration_tests.rs
@@ -34,7 +34,7 @@ fn e2e_build_then_query_pipeline() {
fn e2e_batch_build_and_query() {
let repos = [
("repo-alpha", "db://alpha", "node::alpha"),
- ("repo-beta", "db://beta", "node::beta"),
+ ("repo-beta", "db://beta", "node::beta"),
("repo-gamma", "db://gamma", "node::gamma"),
];