Merge pull request #726 from terraphim/fix/merge-conflicts

fix(terraphim-agent): resolve role shortnames, fix remote thesaurus, add LLM proxy fallback

Author: AlexMikhalev

.docs/summary-Cargo.toml.md

@@ -0,0 +1,19 @@
+# Cargo.toml Summary
+
+## Purpose
+Defines the workspace configuration and dependencies for the Terraphim AI project, managing multiple crates and external dependencies.
+
+## Key Functionality
+- Configures a Rust workspace with multiple member crates
+- Sets up workspace-level dependencies (tokio, reqwest, serde, etc.)
+- Defines profile configurations for different build scenarios (release, ci, etc.)
+- Specifies excluded crates (experimental, Python bindings, desktop-specific)
+- Applies patches for specific dependencies (genai, self_update)
+
+## Important Details
+- Workspace includes crates/, terraphim_server, terraphim_firecracker, terraphim_ai_nodejs
+- Excludes experimental crates like terraphim_agent_application, terraphim_truthforge, etc.
+- Uses edition 2024 and version 1.14.0
+- Configured for async/await with tokio runtime
+- Features conditional compilation via cfg attributes
+- Manages cross-platform builds with specific exclusions

.docs/summary-crates-terraphim_agent-src-client.rs.md

@@ -0,0 +1,24 @@
+# terraphim_agent/src/client.rs Summary
+
+## Purpose
+Provides HTTP client functionality for communicating with the Terraphim server API, handling requests for configuration, chat, document summarization, thesaurus access, and VM management.
+
+## Key Functionality
+- Creates HTTP client with configurable timeouts and user agent
+- Fetches server configuration via GET /config endpoint
+- Resolves role strings to RoleName objects using server config (with fallback)
+- Handles chat interactions with LLM models via POST /chat
+- Summarizes documents via POST /documents/summarize
+- Accesses thesaurus data via GET /thesaurus/{role_name}
+- Provides autocomplete suggestions via GET /autocomplete/{role_name}/{query}
+- Manages VM operations (listing, status, execution, metrics)
+- Supports async document summarization and task management
+
+## Important Details
+- Role resolution falls back to creating RoleName from raw string if not found in config (line 69)
+- Uses async/await pattern with Tokio for non-blocking HTTP requests
+- Implements proper error handling with anyhow::Result
+- Includes configurable timeout via TERRAPHIM_CLIENT_TIMEOUT environment variable
+- Provides both live API methods and dead-code-allowed variants for testing
+- Handles URL encoding for query parameters in various endpoints
+- Supports VM pool management, execution, and monitoring capabilities

.docs/summary-crates-terraphim_agent-src-service.rs.md

@@ -0,0 +1,24 @@
+# terraphim_agent/src/service.rs Summary
+
+## Purpose
+Provides the TUI service layer that manages application state, configuration, and business logic for the Terraphim agent's text-based user interface, coordinating between configuration persistence and the core TerraphimService.
+
+## Key Functionality
+- Manages configuration loading with priority: CLI flag → settings.toml → persistence → embedded defaults
+- Resolves role strings to RoleName objects by searching configuration (name first, then shortname)
+- Provides access to thesaurus data for roles via embedded automata
+- Handles chat interactions with LLM providers based on role configuration
+- Implements document search, extraction, summarization, and connectivity checking
+- Manages role graph operations and topological analysis
+- Supports configuration persistence and reloading from JSON files
+- Provides checklist validation functionality for various domains (code review, security, etc.)
+
+## Important Details
+- Role resolution returns error if role not found in config (line 251), unlike client.rs which falls back to raw string
+- Uses Arc<Mutex<TerraphimService>> for shared state access across async contexts
+- Implements bootstrap-then-persistence pattern for role_config in settings.toml
+- Supports multiple search modes: simple term search and complex SearchQuery with operators
+- Provides thesaurus-backed text operations: extraction, finding matches, autocomplete, fuzzy suggestions
+- Includes connectivity checking for knowledge graph relationships between matched terms
+- Handles device settings with fallback to embedded defaults in sandboxed environments
+- Manages configuration updates and persistence through save_config() method

.docs/summary.md

@@ -1,119 +1,59 @@
-# Terraphim AI - Project Summary
+# Terraphim AI Project Summary
 
 ## Overview
-Terraphim AI is a comprehensive AI agent system designed for semantic knowledge graph search, automation, and intelligent workflow management. The system provides tools for developers and engineers to interact with codebases, documentation, and various data sources through natural language interfaces.
-
-## Key Components Analyzed
-
-### 1. MCP Tool Index (terraphim_agent/src/mcp_tool_index.rs)
-**Purpose**: Implements the Model Context Protocol Tool Index for discovering and searching available MCP tools from configured servers.
-
-**Key Features**:
-- Fast, searchable discovery using Aho-Corasick pattern matching via terraphim_automata
-- Tool management (add, save, load, count)
-- JSON persistence for tool indices
-- Search across tool names, descriptions, and tags
-
-**Performance Characteristics**:
-- Originally required search completion in under 50ms for 100 tools
-- Increased threshold to 70ms to account for system variability while maintaining performance expectations
-- Uses efficient Aho-Corasick automaton for multi-pattern matching
-
-**Recent Fix**:
-- Increased search latency benchmark threshold from 50ms to 70ms to fix intermittent test failures due to system load variability
-
-### 2. TinyClaw Skills Benchmarks (terraphim_tinyclaw/tests/skills_benchmarks.rs)
-**Purpose**: Contains benchmarks for validating the performance of the Terraphim TinyClaw skills system.
-
-**Key Features**:
-- Skill load benchmark (< 100ms NFR)
-- Skill save benchmark (< 50ms NFR)
-- Skill execution benchmark (< 2000ms NFR, increased from 1000ms)
-
-**Recent Fix**:
-- Increased execution time threshold from 1000ms to 2000ms in benchmark_execution_small_skill to fix intermittent test failures due to system load variability while maintaining reasonable performance expectations
-
-## Architecture Analysis
-
-### Strengths
-1. **Modular Design**: Clear separation of concerns between different components (agent, automata, persistence, etc.)
-2. **Performance-Oriented**: Uses efficient algorithms like Aho-Corasick for pattern matching
-3. **Extensible**: Plugin-based architecture for skills and tools
-4. **Well-Tested**: Comprehensive test suite covering unit, integration, and benchmark tests
-5. **Async/Await**: Proper use of Tokio for asynchronous operations where appropriate
-
-### Areas for Improvement
-1. **Benchmark Sensitivity**: Some performance tests are too close to system variability limits
-2. **Resource Management**: Could benefit from more explicit resource cleanup in long-running processes
-3. **Error Handling**: Some error propagation could be more consistent across layers
-
-## Security Analysis
-
-### Strengths
-1. **Input Validation**: Proper validation of inputs in key areas like tool search and skill execution
-2. **Sandboxing**: Execution guards prevent dangerous operations (rm -rf, curl | bash, etc.)
-3. **Path Safety**: Proper handling of file paths to prevent traversal attacks
-4. **Dependency Management**: Uses trusted crates and follows Rust security best practices
-
-### Areas for Improvement
-1. **Dependency Scanning**: Regular dependency vulnerability scanning could be enhanced
-2. **Secrets Management**: While 1Password integration exists, broader secrets management patterns could be documented
-3. **Audit Logging**: More comprehensive audit logging for security-sensitive operations
-
-## Testing Analysis
-
-### Strengths
-1. **Comprehensive Coverage**: Unit tests, integration tests, and performance benchmarks
-2. **Realistic Scenarios**: Tests simulate real-world usage patterns
-3. **Property-Based Testing**: Use of proptest for randomized testing where appropriate
-4. **Benchmark Focus**: Performance benchmarks help catch regressions early
-5. **Test Organization**: Clear separation of different test types
-
-### Areas for Improvement
-1. **Benchmark Flakiness**: Some performance tests are sensitive to system load (addressed by increasing thresholds)
-2. **Test Duration**: Some integration tests take considerable time (>60s)
-3. **Mock Usage**: Good avoidance of mocks in tests as per project policy
-4. **Test Parallelization**: Could benefit from more parallel test execution where safe
-
-## Business Value Analysis
-
-### Key Value Propositions
-1. **Developer Productivity**: Reduces context switching by providing intelligent search and automation
-2. **Knowledge Discovery**: Enables finding relevant information across large codebases and documentation
-3. **Workflow Automation**: Automates repetitive development tasks through skills and agents
-4. **Code Quality**: Helps maintain consistency and discover best practices in codebases
-5. **Onboarding Acceleration**: Helps new team members quickly understand codebases and workflows
-
-### Competitive Advantages
-1. **Semantic Understanding**: Goes beyond keyword search to understand context and intent
-2. **Extensibility**: Easy to add new skills, tools, and data sources
-3. **Privacy-First**: Can operate entirely locally without sending data to external services
-4. **Multi-Modal**: Supports text, code, and potentially other data types
-5. **Integration Friendly**: Designed to work with existing developer tools and workflows
-
-### Target Use Cases
-1. **Code Navigation**: Finding specific implementations, patterns, or usage examples
-2. **Debugging Assistance**: Quickly locating relevant code sections during issue resolution
-3. **Learning & Onboarding**: Helping new team members understand codebase structure
-4. **Refactoring Support**: Finding all usages of specific patterns or APIs
-5. **Automation**: Creating reusable skills for common development tasks
-
-## Recommendations
-
-### Short-Term
-1. Monitor the adjusted benchmark thresholds to ensure they remain appropriate
-2. Consider adding more performance profiling to identify actual bottlenecks
-3. Document the reasoning behind benchmark thresholds for future maintainers
-
-### Medium-Term
-1. Develop more sophisticated performance baselines that account for hardware variations
-2. Consider implementing adaptive benchmarking that adjusts thresholds based on baseline performance
-3. Enhance security audit logging and monitoring capabilities
-
-### Long-Term
-1. Investigate more deterministic performance testing approaches for CI/CD
-2. Consider benchmarking against hardware profiles to set appropriate thresholds
-3. Explore ways to make benchmark tests less sensitive to environmental factors while maintaining their validity
-
-## Conclusion
-The Terraphim AI project demonstrates strong architectural foundations with a focus on performance, security, and extensibility. The recent fixes to benchmark thresholds address test flakiness due to system variability while maintaining meaningful performance expectations. The codebase shows good adherence to Rust best practices and project-specific guidelines, resulting in a reliable and maintainable system that delivers significant value to development teams seeking to improve productivity and code quality.
+Terraphim AI is a Rust-based AI agent system featuring a modular architecture with multiple crates, providing both online (server-connected) and offline (embedded) capabilities for knowledge work, document processing, and AI-assisted tasks.
+
+## Architecture
+- **Workspace Structure**: Cargo workspace with multiple member crates including `terraphim_agent`, `terraphim_server`, `terraphim_firecracker`, and `terraphim_ai_nodejs`
+- **Modular Design**: Separation of concerns between client communication, service logic, and core functionality
+- **Async Runtime**: Built on Tokio for asynchronous operations throughout the codebase
+- **Configuration System**: Multi-layer config loading with priority: CLI flags → settings.toml → persistence → embedded defaults
+
+## Key Components
+
+### Communication Layer (`terraphim_agent/src/client.rs`)
+- HTTP client for server API communication
+- Features configurable timeouts and user agent
+- Role resolution that falls back to creating RoleName from raw string when not found in server config
+- Supports chat, document summarization, thesaurus access, autocomplete, and VM management operations
+
+### Service Layer (`terraphim_agent/src/service.rs`)
+- TUI service managing application state and business logic
+- Coordinates between configuration persistence and core TerraphimService
+- Role resolution that returns error when role not found in config (contrasts with client.rs)
+- Provides thesaurus-backed text operations, search, extraction, summarization, and connectivity checking
+- Implements bootstrap-then-persistence pattern for role configuration
+
+### Dependencies & Tooling
+- Core dependencies: tokio, reqwest, serde, thiserror, anyhow, async-trait
+- Conditional compilation via cfg attributes for feature flags
+- Cross-platform build management with specific exclusions
+- Development tooling includes Clippy, Rustfmt, and custom validation scripts
+
+## Notable Patterns & Characteristics
+1. **Role Resolution Inconsistency**: 
+   - Online mode (client.rs): Falls back to raw role string when not found in config
+   - Offline mode (service.rs): Returns error when role not found in config
+   - This creates different behavior between connected and disconnected states
+
+2. **Configuration Loading**: Sophisticated multi-source configuration system with fallback hierarchy
+
+3. **Error Handling**: Consistent use of `anyhow::Result` and `thiserror` for error propagation
+
+4. **Async/Await**: Extensive use of asynchronous patterns with proper timeout handling
+
+5. **Testing Approach**: Emphasis on integration testing with real services rather than mocks
+
+## Current Focus Areas
+Based on recent documentation and code review activities:
+- Cross-mode consistency between online and offline operations
+- Role resolution and validation improvements
+- Test coverage and compilation fixes
+- Documentation maintenance and accuracy
+- Performance optimization and benchmarking
+
+## Build & Development
+- Standard Rust toolchain with cargo workspace
+- Features: openrouter, mcp-rust-sdk for conditional compilation
+- Profile configurations for release, CI, and LTO-optimized builds
+- Pre-commit hooks for code quality assurance

crates/terraphim_agent/Cargo.toml

@@ -13,7 +13,7 @@ readme = "../../README.md"
 
 [features]
 default = ["repl-interactive"]
-repl = ["dep:rustyline", "dep:colored", "dep:comfy-table", "dep:dirs"]
+repl = ["dep:rustyline", "dep:colored", "dep:comfy-table"]
 repl-interactive = ["repl"]
 # NOTE: repl-sessions re-enabled for local development (path dependency)
 repl-full = ["repl", "repl-chat", "repl-mcp", "repl-file", "repl-custom", "repl-web", "repl-interactive", "repl-sessions"]
@@ -62,7 +62,7 @@ dialoguer = "0.12"  # Interactive CLI prompts for onboarding wizard
 rustyline = { version = "17.0", optional = true }
 colored = { version = "3.0", optional = true }
 comfy-table = { version = "7.0", optional = true }
-dirs = { version = "5.0", optional = true }
+dirs = { version = "5.0" }
 terraphim_types = { path = "../terraphim_types", version = "1.0.0" }
 terraphim_settings = { path = "../terraphim_settings", version = "1.0.0" }
 terraphim_persistence = { path = "../terraphim_persistence", version = "1.0.0" }

crates/terraphim_agent/src/client.rs

@@ -46,6 +46,29 @@ impl ApiClient {
         Ok(body)
     }
 
+    /// Resolve a role string (name or shortname) to a RoleName using server config.
+    /// Falls back to RoleName::new if no match found (server will validate).
+    pub async fn resolve_role(&self, role: &str) -> Result<terraphim_types::RoleName> {
+        use terraphim_types::RoleName;
+        let config_res = self.get_config().await?;
+        let role_lower = role.to_lowercase();
+        let selected = &config_res.config.selected_role;
+        if selected.to_string().to_lowercase() == role_lower {
+            return Ok(selected.clone());
+        }
+        for (name, role_cfg) in &config_res.config.roles {
+            if name.to_string().to_lowercase() == role_lower {
+                return Ok(name.clone());
+            }
+            if let Some(ref sn) = role_cfg.shortname {
+                if sn.to_lowercase() == role_lower {
+                    return Ok(name.clone());
+                }
+            }
+        }
+        Ok(RoleName::new(role))
+    }
+
     pub async fn update_selected_role(&self, role: &str) -> Result<ConfigResponse> {
         let url = format!("{}/config/selected_role", self.base);
         #[derive(Serialize)]

crates/terraphim_agent/src/learnings/capture.rs

@@ -823,13 +823,6 @@ pub enum LearningEntry {
 }
 
 impl LearningEntry {
-    pub fn captured_at(&self) -> DateTime<Utc> {
-        match self {
-            LearningEntry::Learning(l) => l.context.captured_at,
-            LearningEntry::Correction(c) => c.context.captured_at,
-        }
-    }
-
     pub fn source(&self) -> &LearningSource {
         match self {
             LearningEntry::Learning(l) => &l.source,
@@ -845,6 +838,13 @@ impl LearningEntry {
         }
     }
 
+    pub fn captured_at(&self) -> chrono::DateTime<chrono::Utc> {
+        match self {
+            LearningEntry::Learning(l) => l.context.captured_at,
+            LearningEntry::Correction(c) => c.context.captured_at,
+        }
+    }
+
     /// Summary line for display.
     pub fn summary(&self) -> String {
         match self {

crates/terraphim_agent/src/learnings/mod.rs

@@ -36,7 +36,6 @@ pub use capture::{
     capture_correction, capture_failed_command, correct_learning, list_all_entries,
     query_all_entries, suggest_learnings,
 };
-
 // Re-export for testing - not used by CLI yet
 #[allow(unused_imports)]
 pub use capture::{CapturedLearning, LearningContext, LearningError};