feat(pilot): add per-step reasoning support to search algorithms

Add optional per-step reasoning tracking throughout the search pipeline
to provide better context for navigation decisions. This includes:

- Extend build_path_section() to show enhanced navigation history with
  step-by-step reasoning when available, falling back to original
  breadcrumb format otherwise

- Introduce ScoredCandidate struct with optional reasoning field and
  new score_candidates_detailed() function that preserves reasoning
  from the Pilot for each candidate

- Add step_reasons field to SearchState to track reasoning history
  through the search process

- Update beam search algorithm to capture and propagate reasoning
  for each navigation step, including backtracking scenarios

- Modify greedy and MCTS search implementations to support reasoning
  parameters while maintaining backward compatibility

- Enhance SearchPath struct with step_reasons vector to maintain
  per-step reasoning history throughout multi-path algorithms

Author: zTgx

rust/src/retrieval/pilot/builder.rs

@@ -395,7 +395,7 @@ impl ContextBuilder {
         ctx.estimated_tokens += self.estimate_tokens(&ctx.query_section);
 
         // Build path section
-        ctx.path_section = self.build_path_section(state.tree, state.path);
+        ctx.path_section = self.build_path_section(state.tree, state.path, state.step_reasons);
         ctx.estimated_tokens += self.estimate_tokens(&ctx.path_section);
 
         // Build candidates section
@@ -439,7 +439,7 @@ impl ContextBuilder {
         // Show failed path
         ctx.path_section = format!(
             "Failed path:\n{}",
-            self.build_path_section(state.tree, failed_path)
+            self.build_path_section(state.tree, failed_path, None)
         );
         ctx.estimated_tokens += self.estimate_tokens(&ctx.path_section);
 
@@ -463,35 +463,67 @@ impl ContextBuilder {
         format!("User Query:\n{}\n", truncated)
     }
 
-    /// Build current path section.
-    fn build_path_section(&self, tree: &DocumentTree, path: &[NodeId]) -> String {
+    /// Build current path section with optional per-step reasoning.
+    fn build_path_section(
+        &self,
+        tree: &DocumentTree,
+        path: &[NodeId],
+        step_reasons: Option<&[Option<String>]>,
+    ) -> String {
         if path.is_empty() {
             return "Current Position: Root\n".to_string();
         }
 
-        let mut result = String::from("Current Path:\n");
-        result.push_str("Root");
+        let has_reasons = step_reasons
+            .map(|r| r.iter().any(|x| x.is_some()))
+            .unwrap_or(false);
 
-        // Limit depth shown
-        let max_depth = self.effective_max_path_depth();
-        let start = if path.len() > max_depth {
-            path.len() - max_depth
-        } else {
-            0
-        };
+        if !has_reasons {
+            // Original breadcrumb format when no reasoning available
+            let mut result = String::from("Current Path:\n");
+            result.push_str("Root");
+
+            let max_depth = self.effective_max_path_depth();
+            let start = if path.len() > max_depth {
+                path.len() - max_depth
+            } else {
+                0
+            };
 
-        if start > 0 {
-            result.push_str(" → ...");
+            if start > 0 {
+                result.push_str(" → ...");
+            }
+
+            for node_id in path.iter().skip(start) {
+                if let Some(node) = tree.get(*node_id) {
+                    result.push_str(" → ");
+                    result.push_str(&node.title);
+                }
+            }
+
+            result.push('\n');
+            return result;
         }
 
-        for node_id in path.iter().skip(start) {
+        // Enhanced format with per-step reasoning
+        let mut result = String::from("Navigation History:\n");
+        let reasons = step_reasons.unwrap();
+
+        for (i, node_id) in path.iter().enumerate() {
             if let Some(node) = tree.get(*node_id) {
-                result.push_str(" → ");
-                result.push_str(&node.title);
+                let reason = reasons
+                    .get(i)
+                    .and_then(|r| r.as_deref())
+                    .unwrap_or("(automatic selection)");
+                result.push_str(&format!(
+                    "  Step {}: {} — because: {}\n",
+                    i + 1,
+                    node.title,
+                    reason
+                ));
             }
         }
 
-        result.push('\n');
         result
     }
 

rust/src/retrieval/pilot/decision_scorer.rs

@@ -86,18 +86,54 @@ pub async fn score_candidates(
     visited: &HashSet<NodeId>,
     pilot_weight: f32,
     cache: Option<&PilotDecisionCache>,
+    step_reasons: Option<&[Option<String>]>,
 ) -> Vec<(NodeId, f32)> {
+    let scored = score_candidates_detailed(
+        tree, candidates, query, pilot, path, visited, pilot_weight, cache, step_reasons,
+    )
+    .await;
+    scored.into_iter().map(|s| (s.node_id, s.score)).collect()
+}
+
+/// A scored candidate with optional reasoning from the Pilot.
+#[derive(Debug, Clone)]
+pub struct ScoredCandidate {
+    /// The node ID.
+    pub node_id: NodeId,
+    /// Relevance score (0.0 - 1.0).
+    pub score: f32,
+    /// Reason the Pilot chose this node, if available.
+    pub reason: Option<String>,
+}
+
+/// Score child candidates and return detailed results with reasons.
+///
+/// Like [`score_candidates`] but preserves per-candidate reasoning
+/// from the Pilot. Use this when the search algorithm needs to
+/// record why each path step was taken (e.g., for beam search
+/// reasoning history).
+pub async fn score_candidates_detailed(
+    tree: &DocumentTree,
+    candidates: &[NodeId],
+    query: &str,
+    pilot: Option<&dyn Pilot>,
+    path: &[NodeId],
+    visited: &HashSet<NodeId>,
+    pilot_weight: f32,
+    cache: Option<&PilotDecisionCache>,
+    step_reasons: Option<&[Option<String>]>,
+) -> Vec<ScoredCandidate> {
     if candidates.is_empty() {
         return Vec::new();
     }
 
-    // If no Pilot, pure NodeScorer
+    // If no Pilot, pure NodeScorer (no reasons available)
     let Some(p) = pilot else {
-        return score_with_scorer(tree, candidates, query);
+        return score_with_scorer_detailed(tree, candidates, query);
     };
 
     if !p.is_active() {
-        return score_with_scorer(tree, candidates, query);
+        return score_with_scorer_detailed(tree, candidates, query);
     }
 
     // Determine parent node (last in path) for cache key
@@ -109,20 +145,22 @@ pub async fn score_candidates(
             tracing::trace!("Pilot cache hit for parent={:?}", parent);
             cached
         } else {
-            let state = SearchState::new(tree, query, path, candidates, visited);
+            let mut state = SearchState::new(tree, query, path, candidates, visited);
+            state.step_reasons = step_reasons;
             let d = p.decide(&state).await;
             c.put(query, parent, &d).await;
             d
         }
     } else {
-        let state = SearchState::new(tree, query, path, candidates, visited);
+        let mut state = SearchState::new(tree, query, path, candidates, visited);
+        state.step_reasons = step_reasons;
         p.decide(&state).await
     };
 
-    // Build Pilot score map
-    let mut pilot_scores: HashMap<NodeId, f32> = HashMap::new();
+    // Build Pilot score + reason map
+    let mut pilot_data: HashMap<NodeId, (f32, Option<String>)> = HashMap::new();
     for ranked in &decision.ranked_candidates {
-        pilot_scores.insert(ranked.node_id, ranked.score);
+        pilot_data.insert(ranked.node_id, (ranked.score, ranked.reason.clone()));
     }
 
     // Compute NodeScorer fallback scores
@@ -132,24 +170,26 @@ pub async fn score_candidates(
 
     let scorer = NodeScorer::new(ScoringContext::new(query));
 
-    let mut scored: Vec<(NodeId, f32)> = candidates
+    let mut scored: Vec<ScoredCandidate> = candidates
         .iter()
         .map(|&node_id| {
             let algo_score = scorer.score(tree, node_id);
-            let p_score = pilot_scores.get(&node_id).copied().unwrap_or(0.0);
+            let (p_score, reason) = pilot_data.get(&node_id)
+                .map(|(s, r)| (*s, r.clone()))
+                .unwrap_or((0.0, None));
 
-            let final_score = if effective_pilot > 0.0 && pilot_scores.contains_key(&node_id) {
+            let final_score = if effective_pilot > 0.0 && pilot_data.contains_key(&node_id) {
                 (effective_pilot * p_score + scorer_weight * algo_score)
                     / (effective_pilot + scorer_weight)
             } else {
                 algo_score
             };
 
-            (node_id, final_score)
+            ScoredCandidate { node_id, score: final_score, reason }
         })
         .collect();
 
-    scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+    scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));
     scored
 }
 
@@ -163,6 +203,20 @@ fn score_with_scorer(
     scorer.score_and_sort(tree, candidates)
 }
 
+/// Pure NodeScorer fallback returning detailed results (no reasons).
+fn score_with_scorer_detailed(
+    tree: &DocumentTree,
+    candidates: &[NodeId],
+    query: &str,
+) -> Vec<ScoredCandidate> {
+    let scorer = NodeScorer::new(ScoringContext::new(query));
+    scorer
+        .score_and_sort(tree, candidates)
+        .into_iter()
+        .map(|(node_id, score)| ScoredCandidate { node_id, score, reason: None })
+        .collect()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

rust/src/retrieval/pilot/mod.rs

@@ -49,7 +49,7 @@ mod scorer;
 pub use complexity::detect_with_llm;
 pub use config::PilotConfig;
 pub use decision::{InterventionPoint, PilotDecision};
-pub use decision_scorer::{PilotDecisionCache, score_candidates};
+pub use decision_scorer::{PilotDecisionCache, ScoredCandidate, score_candidates, score_candidates_detailed};
 pub use llm_pilot::LlmPilot;
 pub use r#trait::{Pilot, SearchState};
 pub use scorer::{NodeScorer, ScoringContext};

rust/src/retrieval/pilot/trait.rs

@@ -42,6 +42,11 @@ pub struct SearchState<'a> {
     pub best_score: f32,
     /// Whether the search is currently backtracking.
     pub is_backtracking: bool,
+    /// Per-step reasoning for why each node in `path` was chosen.
+    ///
+    /// Same length as `path` when present. `None` means no reasoning
+    /// history is available (e.g. first iteration, algorithm-only mode).
+    pub step_reasons: Option<&'a [Option<String>]>,
 }
 
 impl<'a> SearchState<'a> {
@@ -63,6 +68,7 @@ impl<'a> SearchState<'a> {
             iteration: 0,
             best_score: 0.0,
             is_backtracking: false,
+            step_reasons: None,
         }
     }
 
@@ -78,6 +84,7 @@ impl<'a> SearchState<'a> {
             iteration: 0,
             best_score: 0.0,
             is_backtracking: false,
+            step_reasons: None,
         }
     }
 

rust/src/retrieval/search/beam.rs

@@ -20,7 +20,7 @@ use tracing::debug;
 
 use super::super::RetrievalContext;
 use super::super::types::{NavigationDecision, NavigationStep, SearchPath};
-use crate::retrieval::pilot::{PilotDecisionCache, score_candidates};
+use crate::retrieval::pilot::{PilotDecisionCache, score_candidates, score_candidates_detailed};
 use super::{SearchConfig, SearchResult, SearchTree};
 use crate::document::{DocumentTree, NodeId};
 use crate::retrieval::pilot::{Pilot, SearchState};
@@ -180,6 +180,7 @@ impl BeamSearch {
                     iteration: result.iterations,
                     best_score: result.paths.iter().map(|p| p.score).fold(0.0f32, f32::max),
                     is_backtracking: true,
+                    step_reasons: Some(&entry.path.step_reasons),
                 };
 
                 if let Some(decision) = p.guide_backtrack(&state).await {
@@ -251,7 +252,7 @@ impl BeamSearch {
         let start_children = tree.children(start_node);
         debug!("Start node has {} children", start_children.len());
 
-        let initial_candidates = score_candidates(
+        let initial_candidates = score_candidates_detailed(
             tree,
             &start_children,
             &context.query,
@@ -260,6 +261,7 @@ impl BeamSearch {
             &visited,
             0.7, // Beam: Pilot weight = 0.7
             Some(&cache),
+            None, // No reasoning history at start
         )
         .await;
 
@@ -270,7 +272,14 @@ impl BeamSearch {
         // Split initial candidates into beam and fallback
         let mut sorted_initial: Vec<_> = initial_candidates
             .into_iter()
-            .map(|(node_id, score)| SearchPath::from_node(node_id, score))
+            .map(|s| {
+                let mut path = SearchPath::from_node(s.node_id, s.score);
+                // Record reason for initial selection
+                if let Some(reason) = s.reason {
+                    path.step_reasons = vec![Some(reason)];
+                }
+                path
+            })
             .collect();
         sorted_initial.sort_by(|a, b| {
             b.score
@@ -362,7 +371,7 @@ impl BeamSearch {
                     // Expand this path
                     let children = tree.children(leaf_id);
 
-                    let scored_children = score_candidates(
+                    let scored_children = score_candidates_detailed(
                         tree,
                         &children,
                         &context.query,
@@ -371,23 +380,28 @@ impl BeamSearch {
                         &visited,
                         0.7, // Beam: Pilot weight = 0.7
                         Some(&cache),
+                        Some(&path.step_reasons),
                     )
                     .await;
 
                     if pilot.is_some() && !children.is_empty() {
                         pilot_interventions += 1;
                     }
 
-                    for (child_id, child_score) in scored_children.into_iter().take(beam_width) {
-                        let new_path = path.extend(child_id, child_score);
+                    for sc in scored_children.into_iter().take(beam_width) {
+                        let new_path = if let Some(ref reason) = sc.reason {
+                            path.extend_with_reason(sc.node_id, sc.score, reason)
+                        } else {
+                            path.extend(sc.node_id, sc.score)
+                        };
 
-                        let child_node = tree.get(child_id);
+                        let child_node = tree.get(sc.node_id);
                         result.trace.push(NavigationStep {
-                            node_id: format!("{:?}", child_id),
+                            node_id: format!("{:?}", sc.node_id),
                             title: child_node.map(|n| n.title.clone()).unwrap_or_default(),
-                            score: child_score,
+                            score: sc.score,
                             decision: NavigationDecision::GoToChild(
-                                children.iter().position(|&c| c == child_id).unwrap_or(0),
+                                children.iter().position(|&c| c == sc.node_id).unwrap_or(0),
                             ),
                             depth: child_node.map(|n| n.depth).unwrap_or(0),
                         });
@@ -450,6 +464,7 @@ impl BeamSearch {
                 &visited,
                 0.7,
                 None,
+                None, // No reasoning history for fallback
             )
             .await;
             for (node_id, score) in fallback.into_iter().take(config.top_k) {

rust/src/retrieval/search/greedy.rs

@@ -77,6 +77,7 @@ impl PurePilotSearch {
                 &visited,
                 1.0, // PurePilot: Pilot weight = 1.0
                 Some(&cache),
+                None, // No reasoning history tracked
             )
             .await;