feat(search): add LLM-first search with direct TOC-based location

Add new LLM-first search capability that attempts to directly locate
relevant nodes using the table of contents before falling back to
tree traversal algorithms. The search stage now accepts an LLM client
and implements direct TOC-based node location with structured JSON
responses containing top-3 most relevant entries.

The feature includes:
- New `with_llm_client()` method to configure LLM-based search
- TOC flattening utility for LLM consumption with numbered entries
- Structured JSON prompting for precise node selection
- Proper fallback to beam/greedy search when LLM fails
- Metrics tracking for LLM calls and search performance

refactor(indexer): change default summary strategy to full

Change the default summary strategy from selective to full generation
for improved content indexing quality.

debug: add debug logging throughout indexing and retrieval

Add comprehensive debug logging to track indexing flow, pipeline
options building, summary evaluation, and search operations including
pilot interventions for better debugging visibility.

Author: zTgx

rust/src/client/engine.rs

@@ -214,6 +214,9 @@ impl Engine {
     /// # }
     /// ```
     pub async fn index(&self, ctx: IndexContext) -> Result<String> {
+        println!("Indexing...");
+        println!("ctx: {:?}", ctx);
+        
         let doc = self.indexer.index(ctx).await?;
         let persisted = self.indexer.to_persisted(doc);
 

rust/src/client/indexer.rs

@@ -259,6 +259,8 @@ impl IndexerClient {
         options: &IndexOptions,
         format: DocumentFormat,
     ) -> PipelineOptions {
+        println!("[DEBUG] Building pipeline options for format: {:?} with options: {:?}", format, options);
+
         PipelineOptions {
             mode: match format {
                 DocumentFormat::Markdown => IndexMode::Markdown,
@@ -268,7 +270,8 @@ impl IndexerClient {
             },
             generate_ids: options.generate_ids,
             summary_strategy: if options.generate_summaries {
-                SummaryStrategy::selective(self.config.min_summary_tokens, false)
+                // SummaryStrategy::selective(self.config.min_summary_tokens, false)
+                SummaryStrategy::full()
             } else {
                 SummaryStrategy::none()
             },

rust/src/index/config.rs

@@ -160,7 +160,7 @@ impl Default for PipelineOptions {
         Self {
             mode: IndexMode::Auto,
             generate_ids: true,
-            summary_strategy: SummaryStrategy::default(),
+            summary_strategy: SummaryStrategy::full(),
             thinning: ThinningConfig::default(),
             optimization: OptimizationConfig::default(),
             generate_description: true,

rust/src/index/stages/enhance.rs

@@ -161,6 +161,7 @@ impl IndexStage for EnhanceStage {
                 Some(n) => n.clone(),
                 None => continue,
             };
+            println!("[DEBUG] Evaluating node for summary: {} {}", node.title, node.content);
 
             // Skip if no content
             if node.content.is_empty() {
@@ -204,6 +205,8 @@ impl IndexStage for EnhanceStage {
 
             // Generate summary (generator also has memoization built-in)
             println!("[DEBUG] Calling LLM to generate summary for node: {} ({} tokens)", node.title, token_count);
+            println!("[DEBUG] Node content: {}", node.content);
+
             match generator.generate(&node.title, &node.content).await {
                 Ok(summary) => {
                     if summary.is_empty() {

rust/src/retrieval/pipeline_retriever.rs

@@ -119,7 +119,7 @@ impl PipelineRetriever {
         orchestrator = orchestrator.stage(plan_stage);
 
         // Add search stage with Pilot for semantic navigation
-        let mut search_stage = SearchStage::new();
+        let mut search_stage = SearchStage::new().with_llm_client(self.llm_client.clone());
         if let Some(ref client) = self.llm_client {
             // Create LLM-based Pilot for semantic navigation guidance
             let mut pilot = LlmPilot::new(client.clone(), PilotConfig::default());

rust/src/retrieval/search/beam.rs

@@ -236,6 +236,7 @@ impl SearchTree for BeamSearch {
                                 children.len()
                             );
 
+                            println!("[DEBUG] BEAM SEARCH: Pilot intervening at decision point");
                             match p.decide(&state).await {
                                 decision => {
                                     pilot_interventions += 1;

rust/src/retrieval/search/greedy.rs

@@ -144,6 +144,7 @@ impl SearchTree for GreedySearch {
                         children.len()
                     );
 
+                    println!("[DEBUG] GREEDY SEARCH: Pilot intervening at decision point");
                     match p.decide(&state).await {
                         decision => {
                             pilot_interventions += 1;

rust/src/retrieval/stages/evaluate.rs

@@ -331,6 +331,7 @@ impl RetrievalStage for EvaluateStage {
 
         println!("[DEBUG] EvaluateStage: {} candidates, iteration {}",
             ctx.candidates.len(), ctx.search_iterations);
+        
         info!(
             "Judging sufficiency: {} candidates, iteration {}",
             ctx.candidates.len(),

rust/src/retrieval/stages/search.rs

@@ -6,13 +6,21 @@
 //! This stage executes the selected search algorithm using
 //! the selected retrieval strategy. When a Pilot is provided,
 //! it can provide semantic guidance at key decision points.
+//!
+//! # LLM-First Search
+//!
+//! When an LLM client is provided, the stage will first attempt to
+//! directly locate the top-3 most relevant nodes using the TOC,
+//! falling back to tree traversal algorithms (Beam/Greedy) only if
+//! LLM fails or returns insufficient results.
 
 use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use tracing::{info, warn};
 
-use crate::document::DocumentTree;
-// LlmClient is used via strategy
+use crate::document::{DocumentTree, TocView};
+use crate::llm::LlmClient;
 use crate::retrieval::RetrievalContext; // Legacy context
 use crate::retrieval::pilot::Pilot;
 use crate::retrieval::pipeline::{
@@ -57,6 +65,8 @@ pub struct SearchStage {
     hybrid_strategy: Option<Arc<dyn RetrievalStrategy>>,
     /// Pilot for navigation guidance (optional).
     pilot: Option<Arc<dyn Pilot>>,
+    /// LLM client for direct TOC-based search (optional).
+    llm_client: Option<LlmClient>,
 }
 
 impl Default for SearchStage {
@@ -74,9 +84,20 @@ impl SearchStage {
             semantic_strategy: None,
             hybrid_strategy: None,
             pilot: None,
+            llm_client: None,
         }
     }
 
+    /// Add LLM client for direct TOC-based search.
+    ///
+    /// When provided, the stage will first attempt to locate relevant
+    /// nodes directly using the TOC, falling back to tree traversal
+    /// algorithms only if LLM fails or returns insufficient results.
+    pub fn with_llm_client(mut self, client: Option<LlmClient>) -> Self {
+        self.llm_client = client;
+        self
+    }
+
     /// Add Pilot for semantic navigation guidance.
     ///
     /// When provided, the search algorithm will consult the Pilot
@@ -210,6 +231,172 @@ impl SearchStage {
 
         candidates
     }
+
+    /// Build a flat TOC list for LLM consumption.
+    ///
+    /// Returns a formatted string with numbered entries:
+    /// ```
+    /// [1] Title: "Overview"
+    ///     Summary: "This section covers..."
+    /// [2] Title: "Architecture"
+    ///     Summary: "The system architecture..."
+    /// ```
+    fn build_toc_for_llm(&self, tree: &DocumentTree) -> (String, Vec<crate::document::NodeId>) {
+        let toc_view = TocView::new();
+        let mut entries = Vec::new();
+        let mut node_ids = Vec::new();
+
+        fn collect_entries(
+            tree: &DocumentTree,
+            node_id: crate::document::NodeId,
+            entries: &mut Vec<(usize, String, String)>,
+            node_ids: &mut Vec<crate::document::NodeId>,
+            index: &mut usize,
+        ) {
+            if let Some(node) = tree.get(node_id) {
+                let title = node.title.clone();
+                let summary = if node.summary.is_empty() {
+                    "(no summary)".to_string()
+                } else {
+                    node.summary.clone()
+                };
+                entries.push((*index, title, summary));
+                node_ids.push(node_id);
+                *index += 1;
+
+                for child_id in tree.children(node_id) {
+                    collect_entries(tree, child_id, entries, node_ids, index);
+                }
+            }
+        }
+
+        collect_entries(tree, tree.root(), &mut entries, &mut node_ids, &mut 0);
+
+        let toc_str = entries
+            .iter()
+            .map(|(idx, title, summary)| {
+                format!("[{}] Title: \"{}\"\n    Summary: \"{}\"", idx + 1, title, summary)
+            })
+            .collect::<Vec<_>>()
+            .join("\n\n");
+
+        (toc_str, node_ids)
+    }
+
+    /// Locate top candidates directly via LLM using TOC.
+    ///
+    /// This method bypasses tree traversal by asking the LLM to
+    /// directly identify the most relevant nodes from the TOC.
+    async fn locate_via_llm(
+        &self,
+        query: &str,
+        tree: &DocumentTree,
+    ) -> Option<Vec<CandidateNode>> {
+        let llm_client = self.llm_client.as_ref()?;
+        let (toc_str, node_ids) = self.build_toc_for_llm(tree);
+
+        if node_ids.is_empty() {
+            warn!("No nodes in tree for LLM search");
+            return None;
+        }
+
+        let system_prompt = r#"You are a document navigation assistant. Your task is to locate the most relevant sections in a document hierarchy for a user's query.
+
+CRITICAL INSTRUCTIONS:
+1. Analyze the user query carefully to understand the intent
+2. Examine the provided Table of Contents (TOC) with numbered entries
+3. Select the TOP 3 most relevant entries that would contain the answer
+4. You MUST respond with ONLY valid JSON. No markdown code blocks. No explanations outside JSON.
+
+Your response must have this EXACT structure:
+{
+  "reasoning": "Brief analysis of the query and why you selected these entries",
+  "candidates": [
+    {"node_id": 1, "relevance_score": 0.95, "reason": "Why this entry matches the query"},
+    {"node_id": 2, "relevance_score": 0.80, "reason": "Why this entry is also relevant"},
+    {"node_id": 3, "relevance_score": 0.65, "reason": "Why this entry might be relevant"}
+  ]
+}
+
+Rules:
+- node_id: MUST be a number from the provided TOC (the number in [N] brackets)
+- relevance_score: Number between 0.0 and 1.0 (higher = more relevant)
+- reason: Brief explanation for each selection
+- candidates: Must have exactly 3 items, ordered by relevance (highest first)"#;
+
+        let user_prompt = format!(
+            "USER QUERY: {}\n\nDOCUMENT TOC ({} entries):\n{}\n\nBased on the query and TOC above, select the TOP 3 most relevant entries.\n\nRespond with ONLY the JSON object:",
+            query,
+            node_ids.len(),
+            toc_str
+        );
+
+        info!("Attempting LLM-based search for query: '{}'", query);
+
+        match llm_client.complete(system_prompt, &user_prompt).await {
+            Ok(response) => {
+                // Parse JSON response
+                match serde_json::from_str::<LlmLocateResponse>(&response) {
+                    Ok(llm_response) => {
+                        let mut candidates = Vec::new();
+
+                        for candidate in llm_response.candidates {
+                            // node_id is 1-indexed from LLM, convert to 0-indexed
+                            let idx = candidate.node_id.saturating_sub(1);
+                            if idx < node_ids.len() {
+                                let node_id = node_ids[idx];
+                                if let Some(node) = tree.get(node_id) {
+                                    candidates.push(CandidateNode::new(
+                                        node_id,
+                                        candidate.relevance_score,
+                                        node.depth,
+                                        tree.is_leaf(node_id),
+                                    ));
+                                    info!(
+                                        "LLM selected: [{}] '{}' (score: {:.2})",
+                                        candidate.node_id, node.title, candidate.relevance_score
+                                    );
+                                }
+                            }
+                        }
+
+                        if candidates.is_empty() {
+                            warn!("LLM returned no valid candidates");
+                            return None;
+                        }
+
+                        println!("LLM search found {} candidates", candidates.len());
+                        println!("LLM candidates content: {:?}", candidates);
+                        Some(candidates)
+                    }
+                    Err(e) => {
+                        warn!("Failed to parse LLM response as JSON: {}", e);
+                        warn!("Raw response: {}", response);
+                        None
+                    }
+                }
+            }
+            Err(e) => {
+                warn!("LLM call failed: {}", e);
+                None
+            }
+        }
+    }
+}
+
+/// LLM response for locate query.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct LlmLocateResponse {
+    reasoning: String,
+    candidates: Vec<LlmLocateCandidate>,
+}
+
+/// A candidate from LLM locate response.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct LlmLocateCandidate {
+    node_id: usize,
+    relevance_score: f32,
+    reason: String,
 }
 
 #[async_trait]
@@ -264,6 +451,29 @@ impl RetrievalStage for SearchStage {
         // Increment search iteration
         ctx.increment_search_iteration();
 
+        // === Try LLM-first search (direct TOC-based location) ===
+        if self.llm_client.is_some() {
+            info!("Attempting LLM-first search for query: '{}'", ctx.query);
+
+            if let Some(candidates) = self.locate_via_llm(&ctx.query, &ctx.tree).await {
+                if !candidates.is_empty() {
+                    ctx.candidates = candidates;
+                    ctx.metrics.search_time_ms += start.elapsed().as_millis() as u64;
+                    ctx.metrics.nodes_visited += ctx.candidates.len();
+                    ctx.metrics.llm_calls += 1;
+
+                    info!(
+                        "LLM-first search found {} candidates (skipped tree traversal)",
+                        ctx.candidates.len()
+                    );
+
+                    return Ok(StageOutcome::cont());
+                }
+            }
+
+            info!("LLM-first search returned no results, falling back to tree traversal");
+        }
+
         // Build search config for search algorithms
         let search_config = SearchAlgConfig {
             top_k: config.beam_width * 2,