feat(query): dogfood loop — FileStore persistence + token-budgeted context packs

.github/workflows/rust-ci.yml

@@ -14,4 +14,14 @@ permissions:
 
 jobs:
   rust-ci:
-    uses: hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml@412a7031577112b31ee287cc6060179d638d6500
+    uses: hyperpolymath/standards/.github/workflows/rust-ci-reusable.yml@d135b05bfc647d0c0fbfedc7e80f37ea50f49236
+
+  # The reusable job tests default features only; this keeps the feature-gated
+  # git ingest compiling (it silently broke once — git2 0.21 API drift).
+  feature-git-integration:
+    name: cargo test --features git-integration
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - run: cargo test --features git-integration

.gitignore

@@ -1,3 +1,6 @@
 # Rust build artifacts
 /target/
 Cargo.lock
+
+# Local lattice cache written by `reticulate build`
+.git-reticulator/

.machine_readable/6a2/STATE.a2ml

@@ -8,8 +8,8 @@
 [metadata]
 project = "git-reticulator"
 version = "0.1.0"
-last-updated = "2026-06-04"
-status = "active"   # tidy-up merged (PR #23); migration direction decided 2026-06-04
+last-updated = "2026-07-07"
+status = "active"   # dogfood loop landed 2026-07-07 (FileStore + budgeted query, CLI-wired)
 
 [migration-decision]
 date = "2026-06-04"
@@ -34,14 +34,15 @@ maturity = "experimental"   # experimental | alpha | beta | production | lts
 # ════════════════════════════════════════════════════════════════════════════
 [honest-status]
 lattice-engine = "REAL (src/lattice/mod.rs): Kosaraju SCC-condensation + partial order + LOD zoom (sound+complete) + meet (LCA). cargo green; 10 property/unit tests."
-ingest = "REAL but filesystem-only (src/ingest.rs, std-only, fail-soft). git2 HISTORY ingest NOT yet wired."
-store = "REAL: LatticeStore trait + InMemoryStore default; VeriSimDB octad backend over HTTP behind --features verisim (src/store.rs). Not yet wired into CLI/REST."
-host-reality = "CLI/REST still thin; the compat affine::{build_lattice,query_lattice} shim is now IO-free (no more println-only stubs)."
+ingest = "REAL: std-only filesystem walk (src/ingest.rs, fail-soft) + git2 HEAD-tree/co-change HISTORY ingest behind --features git-integration, wired into the CLI."
+store = "REAL: LatticeStore trait + InMemoryStore + JSON FileStore (versioned envelope, src/store.rs::file) — the default CLI persistence; VeriSimDB octad backend over HTTP behind --features verisim. FileStore wired into CLI build+query 2026-07-07."
+query = "REAL (src/query.rs, 2026-07-07): keyword resolve (case-insensitive, exact-then-coarse ranking) + token-budgeted context packs (chars/4 estimate, drops counted never silent) + text/JSON rendering. This is the dogfood surface — see docs/DOGFOOD.adoc."
+host-reality = "CLI build→query loop REAL end-to-end (ingest → lattice.json → budgeted context pack). REST api still thin; compat affine::{build_lattice,query_lattice} shim retained IO-free."
 embeddings = "NOT WIRED (tch/PyTorch feature off by default)"
 affine-core = "DEFERRED (ADR-006 bridge-first): the Rust engine is the reference core; the .affine core lands after the Rust↔AffineScript bridge."
 proofs = "ZERO mechanized (no .idr/.v/.ads). P2a/P4/P1b are TESTED not proved — see PROOF-NEEDS.md 'Proof status'. 'lattice' = meet-semilattice + digraph (full join not claimed)."
 rust-spark = "Stance documented (docs/decisions/rust-spark-stance.adoc) + spark-theatre-gate.yml added (lenient). Idris2/Zig ABI seam N/A until an FFI surface exists."
-tests = "30 pass (10 new lattice/ingest/store + 20 compat integration/property/api). cargo test exit 0."
+tests = "38+ pass (lattice/ingest/store/query incl. FileStore round-trip + budget-truncation honesty + compat integration/property/api). cargo test exit 0 (2026-07-07)."
 
 [crg]
 grade = "C"
@@ -70,7 +71,8 @@ template-debt = [
 # Core-language question DECIDED 2026-06-04 (ADR-006): AffineScript-first, bridge-first.
 actions = [
   "DELIVERABLE 1 (in affinescript repo, runtime/): build the Rust loader + marshalling for compiled AffineScript — the affine-js equivalent for Rust. Acceptance: a Rust test loads a compiled .affine fn, calls it, round-trips a string via a host extern fn. See docs/MIGRATION-PLAN.adoc.",
-  "LANDED 2026-06-04: Rust reference lattice core (SCC-condensation + partial order + LOD zoom + meet) + filesystem ingest + verisim store seam + 10 tests; cargo green; Rust/SPARK stance + spark-theatre-gate added. NEXT here: wire CLI/REST to ingest→lattice→store, add criterion benches, git2 history ingest.",
+  "LANDED 2026-06-04: Rust reference lattice core (SCC-condensation + partial order + LOD zoom + meet) + filesystem ingest + verisim store seam + 10 tests; cargo green; Rust/SPARK stance + spark-theatre-gate added.",
+  "LANDED 2026-07-07: dogfood loop — JSON FileStore persistence + token-budgeted query engine (src/query.rs) wired into CLI (build writes .git-reticulator/lattice.json, query loads it). NEXT here (docs/DOGFOOD.adoc order): measure token savings on real agent tasks; tree-sitter definition extraction; lattice freshness stamp (HEAD commit); Claude Code skill front-end; criterion benches.",
   "Earn the 'lattice' word: SCC-condensation + partial order in the core, then discharge PROOF-NEEDS.md P1-P2 (Idris2).",
   "Tests/benches: replace smoke-over-stubs with property tests (partial-order laws, zoom soundness/completeness) + criterion benches; strict-but-passable CI with an AffineScript compile gate.",
 ]

README.md

@@ -14,14 +14,15 @@ you can **zoom** into, so an LLM gets the minimal relevant context
 instead of the whole tree.
 
 > [!IMPORTANT]
-> **Maturity: experimental / early skeleton.** The Rust host is ~237 LOC
-> of `println!` stubs; the lattice core lives in
-> `src/lattice/affine/*.affine` (AffineScript) which **cannot compile
-> yet** and, as written, calls Rust crates AffineScript cannot bind.
-> `git2`/`postgres`/embeddings are feature-gated **off**. There are **no
-> proofs** — the word "lattice" is not yet earned (it is currently a
-> typed digraph; see
-> <a href="PROOF-NEEDS.md" class="md">PROOF-NEEDS</a>). Read
+> **Maturity: experimental.** The Rust reference engine is real
+> (SCC condensation, partial order, LOD zoom, containment meet) and the
+> build→query dogfood loop works end-to-end: `reticulate build` persists
+> a lattice file, `reticulate query` returns token-budgeted context
+> packs (see `docs/DOGFOOD.adoc`). The AffineScript core in
+> `src/lattice/affine/*.affine` is **aspirational** (deferred behind the
+> ADR-006 bridge); `postgres`/embeddings are feature-gated **off**;
+> mechanized proofs cover abstract order theory only, not yet the Rust
+> graph (see <a href="PROOF-NEEDS.md" class="md">PROOF-NEEDS</a>). Read
 > `.machine_readable/6a2/STATE.a2ml` for the honest status before
 > relying on anything here.
 
@@ -62,17 +63,22 @@ hallucinated (EXISTENCE). See `.machine_readable/6a2/NEUROSYM.a2ml` and
 # Quickstart
 
 ```bash
-just build                    # cargo build (default features; no git2/db/embeddings)
+cargo build --features git-integration   # git-aware ingest (plain `cargo build` = filesystem walk)
 
 # CLI binary is `reticulate` (subcommands: build | query | api):
-./target/debug/reticulate build --repo /path/to/repo --db postgres://localhost/gr
-./target/debug/reticulate query --zoom auth --db postgres://localhost/gr
+./target/debug/reticulate build --repo /path/to/repo
+#   → ingests the repo, writes /path/to/repo/.git-reticulator/lattice.json
+
+./target/debug/reticulate query --repo /path/to/repo --zoom auth --level definition --budget-tokens 800
+#   → token-budgeted context pack (add --format json for machine consumption)
+
 ./target/debug/reticulate --help
 ```
 
 > [!NOTE]
-> these run today but are **stubs** — `build` prints and returns; it
-> does not yet read the repo or write the DB.
+> `build` and `query` are **real** end-to-end (ingest → lattice file →
+> budgeted context pack; see `docs/DOGFOOD.adoc`). The `api` server and
+> the VeriSimDB store (`--features verisim`) remain thin.
 
 # Architecture
 

docs/DOGFOOD.adoc

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: CC-BY-SA-4.0
+// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell <j.d.a.jewell@open.ac.uk>
+
+= Dogfooding git-reticulator: token-bounded repo context
+:toc:
+:revdate: 2026-07-07
+
+== The point
+
+The owner's primary near-term use case is *reducing LLM token spend* during
+agentic coding sessions: when an agent (Claude Code or similar) starts work on
+a repo, it typically burns thousands of tokens on exploratory `ls`/`grep`/file
+reads just to build a mental map. git-reticulator replaces that exploration
+with one pre-built, queryable structure:
+
+[source,bash]
+----
+# Once per repo (rebuild after large changes; cheap enough to re-run freely):
+reticulate build --repo ~/dev/some-repo
+
+# In-session, instead of grep sweeps:
+reticulate query --repo ~/dev/some-repo --zoom auth --level definition --budget-tokens 800
+reticulate query --repo ~/dev/some-repo --zoom store --level file --format json
+----
+
+`build` ingests the repo (git-aware with `--features git-integration`: HEAD
+tree + commit co-change coupling; plain filesystem walk otherwise) and writes
+`<repo>/.git-reticulator/lattice.json`. `query` resolves a keyword against the
+lattice, zooms each match to the requested level-of-detail, and renders a
+context pack that *fits the stated token budget* — anything dropped is counted
+and reported, never silently omitted.
+
+== How this reduces token count, concretely
+
+1. *Map once, query many.* The lattice is built offline (zero LLM tokens).
+   Each query returns only the relevant subtree at the requested granularity.
+2. *Explicit budget.* `--budget-tokens N` caps the pack (chars/4 estimate), so
+   a context injection can never blow out a prompt.
+3. *Agent integration.* The intended consumption path is a Claude Code skill /
+   `CLAUDE.md` instruction of the form: "before grep-exploring, run
+   `reticulate query --zoom <topic>` and only read the files it names."
+   That converts N speculative file reads into one bounded text block.
+
+== Honest status of the claim
+
+The *mechanism* works end-to-end as of this document's date (build → file →
+query, tested). The *savings* are not yet measured. Before wiring this into
+every session, run the experiment:
+
+* Pick 3 real tasks on a mid-size repo.
+* Run each twice: once with normal agent exploration, once with a
+  "query-the-lattice-first" instruction.
+* Compare input-token counts and whether the agent found the right files.
+
+If the packs lose to plain exploration, the fix is better ingestion (real
+symbol extraction via tree-sitter rather than the current line-prefix
+heuristic), not more infrastructure.
+
+== What is deliberately NOT needed for this loop
+
+* *VeriSimDB / postgres* — the JSON file store is enough for one repo.
+* *Embeddings* — symbolic keyword resolution first; neural similarity later.
+* *AffineScript core* — the Rust reference engine is the core until the
+  bridge lands (ADR-006, tracked in the `affinescript` repo).
+* *Proofs* — PROOF-NEEDS.md governs the "lattice" claim, not the dogfood loop.
+
+== Path to production-ready (proposed order)
+
+1. *Dogfood loop* (this document) — DONE: persistence + budgeted query.
+2. *Measurement* — the experiment above; publish numbers in this file.
+3. *Ingestion quality* — tree-sitter (or per-language) definition extraction;
+   `calls`/`imports` edges, not just containment + co-change.
+4. *Freshness* — record the HEAD commit in the lattice file; `query` warns
+   when the lattice is stale relative to the repo.
+5. *Agent surface* — a Claude Code skill (estate-wide) that fronts `reticulate
+   query`; optionally the REST API for non-CLI consumers.
+6. *CRG B* — lint/fmt/doc-coverage targets per READINESS.md.
+7. *The neuro-symbolic stack* — embeddings, VeriSimDB, proof-carrying
+   retrieval — only after steps 1–5 prove the symbolic half pays for itself.

src/cli/main.rs

@@ -1,8 +1,13 @@
 // SPDX-License-Identifier: MPL-2.0
 // Copyright (c) Jonathan D.A. Jewell <j.d.a.jewell@open.ac.uk>
-use clap::{Parser, Subcommand};
-use git_reticulator::lattice::affine;
+use clap::{Parser, Subcommand, ValueEnum};
+use git_reticulator::lattice::SemanticLevel;
+use git_reticulator::store::file::FileStore;
 use git_reticulator::store::LatticeStore;
+use std::path::PathBuf;
+
+/// Default lattice file location relative to the ingested repo.
+const DEFAULT_LATTICE_REL: &str = ".git-reticulator/lattice.json";
 
 #[derive(Parser)]
 #[command(name = "reticulate")]
@@ -12,25 +17,66 @@ struct Cli {
     command: Commands,
 }
 
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum LevelArg {
+    Module,
+    File,
+    Definition,
+    Block,
+}
+
+impl From<LevelArg> for SemanticLevel {
+    fn from(l: LevelArg) -> Self {
+        match l {
+            LevelArg::Module => SemanticLevel::Module,
+            LevelArg::File => SemanticLevel::File,
+            LevelArg::Definition => SemanticLevel::Definition,
+            LevelArg::Block => SemanticLevel::Block,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum FormatArg {
+    Text,
+    Json,
+}
+
 #[derive(Subcommand)]
 enum Commands {
-    /// Build a semantic lattice from a git repo
+    /// Build a semantic lattice from a repo and persist it to a local lattice
+    /// file (and optionally to VeriSimDB with --features verisim)
     Build {
-        /// Path to the git repository
-        #[arg(short, long)]
+        /// Path to the repository to ingest
+        #[arg(short, long, default_value = ".")]
         repo: String,
-        /// PostgreSQL database URI
+        /// Output lattice file (default: <repo>/.git-reticulator/lattice.json)
         #[arg(short, long)]
-        db: String,
+        out: Option<PathBuf>,
+        /// VeriSimDB base URL (http/https; requires --features verisim)
+        #[arg(short, long)]
+        db: Option<String>,
     },
-    /// Query the lattice with a zoom level to minimize token cost
+    /// Query a built lattice for a token-budgeted context pack
     Query {
-        /// Semantic node or keyword to zoom into
+        /// Keyword to resolve (case-insensitive substring over node names)
         #[arg(short, long)]
         zoom: String,
-        /// PostgreSQL database URI
+        /// Lattice file to query (default: <repo>/.git-reticulator/lattice.json)
         #[arg(short, long)]
-        db: String,
+        lattice: Option<PathBuf>,
+        /// Repository the lattice was built from (locates the default lattice file)
+        #[arg(short, long, default_value = ".")]
+        repo: String,
+        /// Level-of-detail to zoom matches to
+        #[arg(long, value_enum, default_value_t = LevelArg::Definition)]
+        level: LevelArg,
+        /// Output format
+        #[arg(short, long, value_enum, default_value_t = FormatArg::Text)]
+        format: FormatArg,
+        /// Token budget for the rendered context pack (chars/4 estimate)
+        #[arg(short, long, default_value_t = 2000)]
+        budget_tokens: usize,
     },
     /// Start the REST API server for LLM integration
     Api {
@@ -56,13 +102,17 @@ fn reticulate_ingest(repo: &str) -> git_reticulator::lattice::Lattice {
     git_reticulator::ingest::from_path(repo)
 }
 
+fn default_lattice_path(repo: &str) -> PathBuf {
+    PathBuf::from(repo).join(DEFAULT_LATTICE_REL)
+}
+
 #[tokio::main]
 async fn main() {
     let cli = Cli::parse();
     env_logger::init();
 
     match &cli.command {
-        Commands::Build { repo, db } => {
+        Commands::Build { repo, out, db } => {
             println!("🚀 Reticulating {repo} ...");
             let lattice = reticulate_ingest(repo);
             let cond = lattice.condense();
@@ -74,34 +124,66 @@ async fn main() {
                 cond.is_acyclic()
             );
 
+            let out_path = out.clone().unwrap_or_else(|| default_lattice_path(repo));
+            let mut store = FileStore::new(&out_path);
+            match store.persist(&lattice) {
+                Ok(n) => println!("📦 persisted {n} nodes to {}", out_path.display()),
+                Err(e) => {
+                    eprintln!("❌ cannot write {}: {e}", out_path.display());
+                    std::process::exit(1);
+                }
+            }
+
             #[cfg(feature = "verisim")]
-            let to_verisim = if db.starts_with("http://") || db.starts_with("https://") {
-                let store = git_reticulator::store::verisim::VerisimStore::new(db.clone());
-                match store.persist(&lattice).await {
-                    Ok(n) => println!("📦 persisted {n} octads to VeriSimDB ({db})"),
-                    Err(e) => eprintln!("⚠️  verisim persist failed: {e}"),
+            if let Some(db) = db {
+                if db.starts_with("http://") || db.starts_with("https://") {
+                    let store = git_reticulator::store::verisim::VerisimStore::new(db.clone());
+                    match store.persist(&lattice).await {
+                        Ok(n) => println!("📦 persisted {n} octads to VeriSimDB ({db})"),
+                        Err(e) => eprintln!("⚠️  verisim persist failed: {e}"),
+                    }
                 }
-                true
-            } else {
-                false
-            };
+            }
             #[cfg(not(feature = "verisim"))]
-            let to_verisim = false;
-
-            if !to_verisim {
-                let mut store = git_reticulator::store::InMemoryStore::new();
-                let n = match store.persist(&lattice) {
-                    Ok(n) => n,
-                    // InMemoryStore is Infallible — this arm is unreachable.
-                    Err(never) => match never {},
-                };
-                println!("📦 persisted {n} nodes to the in-memory store (target: {db})");
+            if let Some(db) = db {
+                eprintln!("⚠️  --db {db} ignored: rebuild with --features verisim");
             }
+
             println!("✅ done.");
         }
-        Commands::Query { zoom, db } => {
-            println!("🔍 Querying lattice for context: {}", zoom);
-            affine::query_lattice(zoom, db);
+        Commands::Query {
+            zoom,
+            lattice,
+            repo,
+            level,
+            format,
+            budget_tokens,
+        } => {
+            let path = lattice
+                .clone()
+                .unwrap_or_else(|| default_lattice_path(repo));
+            let lat = match FileStore::load(&path) {
+                Ok(lat) => lat,
+                Err(e) => {
+                    eprintln!(
+                        "❌ {e}\n   no usable lattice at {} — run `reticulate build --repo {repo}` first",
+                        path.display()
+                    );
+                    std::process::exit(1);
+                }
+            };
+            let result =
+                git_reticulator::query::context_pack(&lat, zoom, (*level).into(), *budget_tokens);
+            match format {
+                FormatArg::Text => print!("{}", git_reticulator::query::render_text(&result)),
+                FormatArg::Json => match serde_json::to_string_pretty(&result) {
+                    Ok(json) => println!("{json}"),
+                    Err(e) => {
+                        eprintln!("❌ cannot serialize result: {e}");
+                        std::process::exit(1);
+                    }
+                },
+            }
         }
         Commands::Api { db } => {
             println!("🌐 Starting Git-Reticulator API on http://localhost:8080");