feat: add memo store for caching LLM decisions

- Introduce MemoStore for caching LLM decisions based on context fingerprints
- Add memo store integration to EngineBuilder with with_memo_store method
- Implement cache statistics snapshot and invalidation methods in MemoStore
- Add pilot decision caching functionality in LlmPilot with context fingerprinting
- Support batch invalidation by operation type and model prefix
- Enable memo store integration in PipelineRetriever

Author: zTgx

src/client/builder.rs

@@ -36,6 +36,7 @@
 use std::path::PathBuf;
 
 use crate::config::{Config, ConfigLoader, RetrievalConfig};
+use crate::memo::MemoStore;
 use crate::retrieval::PipelineRetriever;
 use crate::storage::Workspace;
 
@@ -108,6 +109,9 @@ pub struct EngineBuilder {
 
     /// Precise mode flag.
     precise_mode: bool,
+
+    /// Memo store for caching LLM decisions.
+    memo_store: Option<MemoStore>,
 }
 
 impl EngineBuilder {
@@ -126,6 +130,7 @@ impl EngineBuilder {
             top_k: None,
             fast_mode: false,
             precise_mode: false,
+            memo_store: None,
         }
     }
 
@@ -192,6 +197,39 @@ impl EngineBuilder {
         self
     }
 
+    /// Set a memo store for caching LLM decisions.
+    ///
+    /// When enabled, the pilot will cache navigation decisions based on
+    /// context fingerprints, avoiding redundant API calls for similar
+    /// navigation scenarios.
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use vectorless::client::EngineBuilder;
+    /// use vectorless::memo::MemoStore;
+    /// use chrono::Duration;
+    ///
+    /// # #[tokio::main]
+    /// # async fn main() -> Result<(), vectorless::BuildError> {
+    /// let memo_store = MemoStore::new()
+    ///     .with_ttl(Duration::days(7))
+    ///     .with_model("gpt-4o");
+    ///
+    /// let engine = EngineBuilder::new()
+    ///     .with_workspace("./data")
+    ///     .with_memo_store(memo_store)
+    ///     .build()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    #[must_use]
+    pub fn with_memo_store(mut self, store: MemoStore) -> Self {
+        self.memo_store = Some(store);
+        self
+    }
+
     // ============================================================
     // LLM Configuration
     // ============================================================

src/memo/mod.rs

@@ -32,4 +32,4 @@ mod store;
 mod types;
 
 pub use store::MemoStore;
-pub use types::{MemoEntry, MemoKey, MemoOpType, MemoValue, MemoStats};
+pub use types::{MemoEntry, MemoKey, MemoOpType, MemoStats, MemoValue, PilotDecisionValue};

src/memo/store.rs

@@ -7,7 +7,9 @@
 
 use std::collections::HashMap;
 use std::future::Future;
+use std::num::NonZeroUsize;
 use std::path::Path;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 
 use chrono::{Duration, Utc};
@@ -17,7 +19,7 @@ use serde::{Deserialize, Serialize};
 use tokio::sync::RwLock as AsyncRwLock;
 use tracing::{debug, info, warn};
 
-use super::types::{MemoEntry, MemoKey, MemoStats, MemoValue};
+use super::types::{MemoEntry, MemoKey, MemoOpType, MemoStats, MemoValue};
 use crate::error::Result;
 use crate::utils::fingerprint::Fingerprint;
 
@@ -246,6 +248,99 @@ impl MemoStore {
         result
     }
 
+    /// Get cache statistics synchronously.
+    ///
+    /// This acquires a read lock on the stats, which is generally fast.
+    /// Use this when you need stats without async context.
+    pub fn stats_snapshot(&self) -> MemoStats {
+        // Use try_read to avoid blocking; fall back to defaults if locked
+        match self.stats.try_read() {
+            Ok(stats) => {
+                let mut result = stats.clone();
+                result.entries = self.len();
+                result
+            }
+            Err(_) => MemoStats {
+                entries: self.len(),
+                ..Default::default()
+            },
+        }
+    }
+
+    /// Invalidate all entries of a specific operation type.
+    ///
+    /// This is useful for batch invalidation when the algorithm for
+    /// a specific operation type changes.
+    ///
+    /// # Example
+    ///
+    /// ```rust,ignore
+    /// // Invalidate all pilot decision caches
+    /// let removed = store.invalidate_by_op_type(MemoOpType::PilotDecision);
+    /// println!("Removed {} cached pilot decisions", removed);
+    /// ```
+    pub fn invalidate_by_op_type(&self, op_type: MemoOpType) -> usize {
+        let mut cache = self.cache.write();
+        let before = cache.len();
+
+        // Collect keys to remove based on entry value type
+        let keys_to_remove: Vec<String> = cache
+            .iter()
+            .filter_map(|(key, entry)| {
+                let matches = match (&entry.value, op_type) {
+                    (MemoValue::Summary(_), MemoOpType::Summary) => true,
+                    (MemoValue::PilotDecision(_), MemoOpType::PilotDecision) => true,
+                    (MemoValue::QueryAnalysis(_), MemoOpType::QueryAnalysis) => true,
+                    (MemoValue::Extraction(_), MemoOpType::Extraction) => true,
+                    _ => false,
+                };
+                if matches { Some(key.clone()) } else { None }
+            })
+            .collect();
+
+        // Remove entries
+        for key in keys_to_remove {
+            cache.pop(&key);
+        }
+
+        let removed = before - cache.len();
+        if removed > 0 {
+            debug!("Invalidated {} entries for op_type {:?}", removed, op_type);
+        }
+        removed
+    }
+
+    /// Invalidate all entries matching a model ID prefix.
+    ///
+    /// This is useful when switching models or when a model's behavior changes.
+    ///
+    /// # Example
+    ///
+    /// ```rust,ignore
+    /// // Invalidate all GPT-4 caches
+    /// let removed = store.invalidate_by_model_prefix("gpt-4");
+    /// ```
+    pub fn invalidate_by_model_prefix(&self, prefix: &str) -> usize {
+        let mut cache = self.cache.write();
+        let before = cache.len();
+
+        // Since the key is a fingerprint, we need to check model_id from entries
+        // For now, we'll clear all entries if prefix matches our model_id
+        // A better approach would be to store model_id in entry metadata
+        let should_clear = self.model_id.as_ref()
+            .map(|m| m.starts_with(prefix))
+            .unwrap_or(false);
+
+        if should_clear {
+            cache.clear();
+            let removed = before;
+            debug!("Invalidated all {} entries (model prefix '{}')", removed, prefix);
+            return removed;
+        }
+
+        0
+    }
+
     /// Remove expired entries.
     pub fn prune_expired(&self) -> usize {
         let mut cache = self.cache.write();

src/retrieval/pilot/llm_pilot.rs

@@ -12,7 +12,9 @@ use tracing::{debug, info, warn};
 
 use crate::document::DocumentTree;
 use crate::llm::{LlmClient, LlmExecutor};
+use crate::memo::{MemoKey, MemoOpType, MemoStore, MemoValue};
 use crate::throttle::ConcurrencyController;
+use crate::utils::fingerprint::Fingerprint;
 
 use super::budget::BudgetController;
 use super::builder::ContextBuilder;
@@ -43,6 +45,11 @@ use super::r#trait::{Pilot, SearchState};
 /// │  │ Budget      │  │ LlmExecutor           │                │
 /// │  │ Controller  │  │ (throttle+retry+fall) │                │
 /// │  └─────────────┘  └───────────────────────┘                │
+/// │                                                              │
+/// │  ┌─────────────┐  ┌───────────────────────┐                │
+/// │  │ Memo        │  │ (cache LLM decisions) │                │
+/// │  │ Store       │  │                       │                │
+/// │  └─────────────┘  └───────────────────────┘                │
 /// └─────────────────────────────────────────────────────────────┘
 /// ```
 ///
@@ -81,6 +88,8 @@ pub struct LlmPilot {
     response_parser: ResponseParser,
     /// Feedback learner for improving decisions (optional).
     learner: Option<Arc<PilotLearner>>,
+    /// Memo store for caching decisions (optional).
+    memo_store: Option<MemoStore>,
 }
 
 impl std::fmt::Debug for LlmPilot {
@@ -107,6 +116,7 @@ impl LlmPilot {
             prompt_builder: PromptBuilder::new(),
             response_parser: ResponseParser::new(),
             learner: None,
+            memo_store: None,
         }
     }
 
@@ -126,6 +136,7 @@ impl LlmPilot {
             prompt_builder: PromptBuilder::new(),
             response_parser: ResponseParser::new(),
             learner: None,
+            memo_store: None,
         }
     }
 
@@ -144,6 +155,7 @@ impl LlmPilot {
             prompt_builder: PromptBuilder::new(),
             response_parser: ResponseParser::new(),
             learner: None,
+            memo_store: None,
         }
     }
 
@@ -165,6 +177,7 @@ impl LlmPilot {
             prompt_builder,
             response_parser: ResponseParser::new(),
             learner: None,
+            memo_store: None,
         }
     }
 
@@ -186,6 +199,16 @@ impl LlmPilot {
         self
     }
 
+    /// Add a memo store for caching decisions.
+    ///
+    /// When enabled, the pilot will cache LLM decisions based on
+    /// context fingerprints, avoiding redundant API calls for
+    /// similar navigation scenarios.
+    pub fn with_memo_store(mut self, store: MemoStore) -> Self {
+        self.memo_store = Some(store);
+        self
+    }
+
     /// Check if using LlmExecutor (unified throttle/retry/fallback).
     pub fn has_executor(&self) -> bool {
         self.executor.is_some()
@@ -196,11 +219,21 @@ impl LlmPilot {
         self.learner.is_some()
     }
 
+    /// Check if using memo store.
+    pub fn has_memo_store(&self) -> bool {
+        self.memo_store.is_some()
+    }
+
     /// Get the feedback learner (if any).
     pub fn learner(&self) -> Option<&PilotLearner> {
         self.learner.as_deref()
     }
 
+    /// Get the memo store (if any).
+    pub fn memo_store(&self) -> Option<&MemoStore> {
+        self.memo_store.as_ref()
+    }
+
     /// Record feedback for a decision.
     pub fn record_feedback(&self, record: FeedbackRecord) {
         if let Some(ref learner) = self.learner {
@@ -210,6 +243,18 @@ impl LlmPilot {
         }
     }
 
+    /// Compute a cache key for a pilot decision.
+    fn compute_cache_key(&self, context: &super::builder::PilotContext, point: InterventionPoint) -> Option<MemoKey> {
+        let store = self.memo_store.as_ref()?;
+
+        // Build a fingerprint from the context using available methods
+        let context_str = context.to_string();
+        let context_fp = Fingerprint::from_str(&context_str);
+        let query_fp = Fingerprint::from_str(&context.query_section);
+
+        Some(MemoKey::pilot_decision(&context_fp, &query_fp))
+    }
+
     /// Check if budget allows LLM calls.
     fn has_budget(&self) -> bool {
         self.budget.can_call()
@@ -243,6 +288,20 @@ impl LlmPilot {
         context: &super::builder::PilotContext,
         candidates: &[crate::document::NodeId],
     ) -> PilotDecision {
+        // Check memo cache first
+        if let Some(ref store) = self.memo_store {
+            if let Some(cache_key) = self.compute_cache_key(context, point) {
+                if let Some(cached) = store.get(&cache_key) {
+                    if let MemoValue::PilotDecision(decision_value) = cached {
+                        debug!("Memo cache hit for pilot decision at {:?}", point);
+                        // Convert cached value back to PilotDecision
+                        let decision = self.cached_value_to_decision(decision_value, candidates, point);
+                        return decision;
+                    }
+                }
+            }
+        }
+
         // Build prompt
         let prompt = self.prompt_builder.build(point, context);
 
@@ -313,6 +372,16 @@ impl LlmPilot {
                     decision.ranked_candidates.len()
                 );
 
+                // Cache the decision
+                if let Some(ref store) = self.memo_store {
+                    if let Some(cache_key) = self.compute_cache_key(context, point) {
+                        let decision_value = self.decision_to_cached_value(&decision);
+                        let tokens_saved = prompt.estimated_tokens as u64 + output_tokens as u64;
+                        store.put_with_tokens(cache_key, MemoValue::PilotDecision(decision_value), tokens_saved);
+                        debug!("Memo cache stored for pilot decision at {:?}", point);
+                    }
+                }
+
                 decision
             }
             Err(e) => {
@@ -322,6 +391,45 @@ impl LlmPilot {
         }
     }
 
+    /// Convert a PilotDecision to a cacheable value.
+    fn decision_to_cached_value(&self, decision: &PilotDecision) -> crate::memo::PilotDecisionValue {
+        crate::memo::PilotDecisionValue {
+            selected_idx: decision.ranked_candidates.first()
+                .map(|c| c.node_id.0.into())
+                .unwrap_or(0),
+            confidence: decision.confidence,
+            reasoning: decision.reasoning.clone(),
+        }
+    }
+
+    /// Convert a cached value back to a PilotDecision.
+    fn cached_value_to_decision(
+        &self,
+        value: crate::memo::PilotDecisionValue,
+        candidates: &[crate::document::NodeId],
+        point: InterventionPoint,
+    ) -> PilotDecision {
+        let ranked = candidates
+            .iter()
+            .enumerate()
+            .map(|(i, &node_id)| super::decision::RankedCandidate {
+                node_id,
+                score: if i == value.selected_idx { 1.0 } else { 0.5 / (i + 1) as f32 },
+                reason: None,
+            })
+            .collect();
+
+        PilotDecision {
+            ranked_candidates: ranked,
+            direction: super::decision::SearchDirection::GoDeeper {
+                reason: "Cached decision".to_string(),
+            },
+            confidence: value.confidence,
+            reasoning: value.reasoning,
+            intervention_point: point,
+        }
+    }
+
     /// Create a default decision when LLM fails.
     fn default_decision(
         &self,