added support for multiple llm models and removed unavailable tool log

Author: Hakan Bogan

agents/base.py

@@ -6,31 +6,69 @@
 from time import time
 from typing import Any, Dict, List
 
+from langchain_core.callbacks import BaseCallbackHandler
+from langchain_anthropic import ChatAnthropic
 from langchain_openai import ChatOpenAI
 
 from app.config import Settings
 from app.logging import get_logger
-from domain import AgentDecision, AgentRole, Evidence, Finding, FindingType, PRContext, Severity
+from domain import AgentDecision, AgentRole, Evidence, Finding, FindingType, LLMProvider, PRContext, Severity
 
 logger = get_logger(__name__)
 
 
+class TokenUsageCallback(BaseCallbackHandler):
+    """Callback to track token usage from LLM calls."""
+
+    def __init__(self):
+        super().__init__()
+        self.total_input_tokens = 0
+        self.total_output_tokens = 0
+
+    def on_llm_end(self, response, **kwargs):
+        """Capture token usage from LLM response."""
+        if hasattr(response, "llm_output") and response.llm_output:
+            usage = response.llm_output.get("token_usage", {})
+            self.total_input_tokens += usage.get("prompt_tokens", 0)
+            self.total_output_tokens += usage.get("completion_tokens", 0)
+        # Also check response.usage_metadata for newer LangChain versions
+        elif hasattr(response, "usage_metadata"):
+            if hasattr(response.usage_metadata, "input_tokens"):
+                self.total_input_tokens += response.usage_metadata.input_tokens
+            if hasattr(response.usage_metadata, "output_tokens"):
+                self.total_output_tokens += response.usage_metadata.output_tokens
+
+    @property
+    def total_tokens(self) -> int:
+        """Get total tokens used."""
+        return self.total_input_tokens + self.total_output_tokens
+
+
 class BaseAgent(ABC):
     """Abstract base class for all review agents."""
 
     def __init__(self, role: AgentRole, settings: Settings):
         self.role = role
         self.settings = settings
         self.prompt_version = settings.default_prompt_version
+        self.token_callback = TokenUsageCallback()
         self.llm = self._create_llm(settings)
 
-    def _create_llm(self, settings: Settings) -> ChatOpenAI:
-        """Create LLM instance with consistent configuration."""
+    def _create_llm(self, settings: Settings) -> ChatOpenAI | ChatAnthropic:
+        """Create LLM instance with consistent configuration based on provider."""
+        if settings.llm_provider == LLMProvider.ANTHROPIC:
+            return ChatAnthropic(
+                api_key=settings.anthropic_api_key,
+                model=settings.anthropic_model,
+                temperature=settings.openai_temperature,
+                callbacks=[self.token_callback],
+            )
         return ChatOpenAI(
             api_key=settings.openai_api_key,
             model=settings.openai_model,
             temperature=settings.openai_temperature,
             seed=settings.openai_seed,
+            callbacks=[self.token_callback],
         )
 
     @abstractmethod
@@ -72,6 +110,15 @@ def _execute_with_timing(self, func, *args, **kwargs):
         result = func(*args, **kwargs)
         return result, time() - start
 
+    def _reset_token_tracking(self) -> None:
+        """Reset token callback counters."""
+        self.token_callback.total_input_tokens = 0
+        self.token_callback.total_output_tokens = 0
+
+    def _get_token_count(self) -> int:
+        """Get total tokens used from callback."""
+        return self.token_callback.total_tokens
+
     def _validate_findings(self, findings: List[Finding]) -> List[Finding]:
         """Validate findings meet evidence requirements."""
         valid = []

agents/change_context_analyst.py

@@ -76,12 +76,24 @@ def _build_analysis_context(self, context: PRContext, prompt_template: str) -> s
     def _execute_task(self, agent: Agent, task: Task) -> dict:
         """Execute CrewAI task and extract results."""
         try:
-            crew = Crew(agents=[agent], tasks=[task], verbose=False)
+            self._reset_token_tracking()
+            # Pass callback to Crew to track token usage
+            crew = Crew(
+                agents=[agent],
+                tasks=[task],
+                verbose=False,
+                callbacks=[self.token_callback]
+            )
             result = crew.kickoff()
             output = str(result) if result else ""
+            tokens = self._get_token_count()
+            # Fallback: if callback didn't capture tokens, try to get from result
+            if tokens == 0 and hasattr(result, 'usage_metadata'):
+                if hasattr(result.usage_metadata, 'total_tokens'):
+                    tokens = result.usage_metadata.total_tokens
             return {
                 "reasoning": output[:500] or "Analysis completed",
-                "tokens": 0,
+                "tokens": tokens,
                 "raw_output": output,
             }
         except Exception as e:

agents/documentation_reviewer.py

@@ -79,10 +79,20 @@ def _build_analysis_context(self, context: PRContext, prompt_template: str) -> s
     def _execute_task(self, agent: Agent, task: Task) -> dict:
         """Execute CrewAI task."""
         try:
-            crew = Crew(agents=[agent], tasks=[task], verbose=False)
+            self._reset_token_tracking()
+            crew = Crew(
+                agents=[agent],
+                tasks=[task],
+                verbose=False,
+                callbacks=[self.token_callback]
+            )
             result = crew.kickoff()
             output = str(result) if result else ""
-            return {"reasoning": output[:500], "tokens": 0, "raw_output": output}
+            tokens = self._get_token_count()
+            if tokens == 0 and hasattr(result, 'usage_metadata'):
+                if hasattr(result.usage_metadata, 'total_tokens'):
+                    tokens = result.usage_metadata.total_tokens
+            return {"reasoning": output[:500], "tokens": tokens, "raw_output": output}
         except Exception as e:
             logger.error(f"Error executing CrewAI task: {e}")
             return {"reasoning": f"Analysis error: {e}", "tokens": 0, "raw_output": ""}

agents/logic_reviewer.py

@@ -76,12 +76,22 @@ def _build_analysis_context(self, context: PRContext, prompt_template: str) -> s
     def _execute_task(self, agent: Agent, task: Task) -> dict:
         """Execute CrewAI task and extract results."""
         try:
-            crew = Crew(agents=[agent], tasks=[task], verbose=False)
+            self._reset_token_tracking()
+            crew = Crew(
+                agents=[agent],
+                tasks=[task],
+                verbose=False,
+                callbacks=[self.token_callback]
+            )
             result = crew.kickoff()
             output = str(result) if result else ""
+            tokens = self._get_token_count()
+            if tokens == 0 and hasattr(result, 'usage_metadata'):
+                if hasattr(result.usage_metadata, 'total_tokens'):
+                    tokens = result.usage_metadata.total_tokens
             return {
                 "reasoning": output[:500] or "Analysis completed",
-                "tokens": 0,
+                "tokens": tokens,
                 "raw_output": output,
             }
         except Exception as e:

agents/performance_reviewer.py

@@ -164,10 +164,20 @@ def _build_analysis_context(self, context: PRContext, prompt_template: str) -> s
     def _execute_task(self, agent: Agent, task: Task) -> dict:
         """Execute CrewAI task."""
         try:
-            crew = Crew(agents=[agent], tasks=[task], verbose=False)
+            self._reset_token_tracking()
+            crew = Crew(
+                agents=[agent],
+                tasks=[task],
+                verbose=False,
+                callbacks=[self.token_callback]
+            )
             result = crew.kickoff()
             output = str(result) if result else ""
-            return {"reasoning": output[:500], "tokens": 0, "raw_output": output}
+            tokens = self._get_token_count()
+            if tokens == 0 and hasattr(result, 'usage_metadata'):
+                if hasattr(result.usage_metadata, 'total_tokens'):
+                    tokens = result.usage_metadata.total_tokens
+            return {"reasoning": output[:500], "tokens": tokens, "raw_output": output}
         except Exception as e:
             logger.error(f"Error executing CrewAI task: {e}")
             return {"reasoning": f"Analysis error: {e}", "tokens": 0, "raw_output": ""}

agents/revision_proposer.py

@@ -128,10 +128,20 @@ def _generate_single_patch(
     def _execute_task(self, agent: Agent, task: Task) -> dict:
         """Execute CrewAI task."""
         try:
-            crew = Crew(agents=[agent], tasks=[task], verbose=False)
+            self._reset_token_tracking()
+            crew = Crew(
+                agents=[agent],
+                tasks=[task],
+                verbose=False,
+                callbacks=[self.token_callback]
+            )
             result = crew.kickoff()
             output = str(result) if result else ""
-            return {"reasoning": output[:500], "tokens": 0, "raw_output": output}
+            tokens = self._get_token_count()
+            if tokens == 0 and hasattr(result, 'usage_metadata'):
+                if hasattr(result.usage_metadata, 'total_tokens'):
+                    tokens = result.usage_metadata.total_tokens
+            return {"reasoning": output[:500], "tokens": tokens, "raw_output": output}
         except Exception as e:
             logger.error(f"Error executing CrewAI task: {e}")
             return {"reasoning": f"Error: {e}", "tokens": 0, "raw_output": ""}

agents/test_reviewer.py

@@ -210,10 +210,20 @@ def _build_analysis_context(self, context: PRContext, prompt_template: str) -> s
     def _execute_task(self, agent: Agent, task: Task) -> dict:
         """Execute CrewAI task."""
         try:
-            crew = Crew(agents=[agent], tasks=[task], verbose=False)
+            self._reset_token_tracking()
+            crew = Crew(
+                agents=[agent],
+                tasks=[task],
+                verbose=False,
+                callbacks=[self.token_callback]
+            )
             result = crew.kickoff()
             output = str(result) if result else ""
-            return {"reasoning": output[:500], "tokens": 0, "raw_output": output}
+            tokens = self._get_token_count()
+            if tokens == 0 and hasattr(result, 'usage_metadata'):
+                if hasattr(result.usage_metadata, 'total_tokens'):
+                    tokens = result.usage_metadata.total_tokens
+            return {"reasoning": output[:500], "tokens": tokens, "raw_output": output}
         except Exception as e:
             logger.error(f"Error executing CrewAI task: {e}")
             return {"reasoning": f"Analysis error: {e}", "tokens": 0, "raw_output": ""}

app/cli.py

@@ -16,7 +16,7 @@
 from app.config import get_settings
 from app.logging import setup_logging
 from app.review_storage import ReviewStorage
-from domain import PRMetadata, SystemType
+from domain import Language, PRMetadata, SystemType
 from eval import ComparisonAnalyzer, EvaluationRunner
 from flows import ReviewFlow
 
@@ -263,7 +263,8 @@ def review(
     title: Optional[str] = typer.Option(None, "--title", help="PR title (auto-fetched from GitHub if not provided)"),
     description: Optional[str] = typer.Option(
         None, "--description", help="PR description (auto-fetched from GitHub if not provided)"),
-    language: str = typer.Option("python", "--language", help="Primary language"),
+    language: str = typer.Option("python", "--language",
+                                 help="Primary language (python, javascript, typescript, etc.)"),
     multi_agent: bool = typer.Option(True, "--multi-agent/--single-agent", help="Use multi-agent system"),
     output: Optional[Path] = typer.Option(None, "--output", "-o", help="Output file path"),
     checkout_pr: bool = typer.Option(True, "--checkout-pr/--no-checkout-pr",
@@ -321,6 +322,20 @@ def review(
             console.print(f"[yellow]Warning: Could not checkout PR branch: {e}[/yellow]")
             console.print("[yellow]Continuing with current branch...[/yellow]")
 
+    # Convert language string to Language enum
+    try:
+        language_enum = Language(language.lower())
+    except ValueError:
+        # Try to map common aliases
+        language_map = {
+            "js": Language.JAVASCRIPT,
+            "ts": Language.TYPESCRIPT,
+            "c++": Language.CPP,
+            "c#": Language.CSHARP,
+        }
+        language_enum = language_map.get(language.lower(), Language.PYTHON)
+        console.print(f"[yellow]Warning: Unknown language '{language}', defaulting to {language_enum.value}[/yellow]")
+
     # Create PR metadata
     pr_metadata = PRMetadata(
         pr_id=pr_id,
@@ -330,11 +345,11 @@ def review(
         title=title,
         description=description,
         author=author,
-        language=language,
+        language=language_enum,
     )
 
     # Run review
-    flow = ReviewFlow(settings)
+    flow = ReviewFlow(settings, language=language_enum)
 
     try:
         if multi_agent:

app/config.py

@@ -4,9 +4,11 @@
 from pathlib import Path
 from typing import Literal
 
-from pydantic import Field, field_validator
+from pydantic import Field, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+from domain import LLMProvider
+
 
 def _find_project_root() -> Path:
     """Find project root directory by looking for .env file or pyproject.toml."""
@@ -42,7 +44,11 @@ class Settings(BaseSettings):
     )
 
     # LLM Configuration
-    openai_api_key: str = Field(..., description="OpenAI API key")
+    llm_provider: LLMProvider = Field(
+        default=LLMProvider.OPENAI,
+        description="LLM provider to use"
+    )
+    openai_api_key: str | None = Field(default=None, description="OpenAI API key")
     openai_model: str = Field(default="gpt-4-turbo-preview", description="OpenAI model")
     openai_temperature: float = Field(default=0.0, ge=0.0, le=2.0, description="LLM temperature")
     openai_seed: int = Field(default=42, description="Seed for deterministic outputs")
@@ -154,6 +160,15 @@ def validate_s3_config(cls, v: str, info: any) -> str:
                 )
         return v
 
+    @model_validator(mode="after")
+    def validate_llm_provider_config(self) -> "Settings":
+        """Validate that the appropriate API key exists for the selected provider."""
+        if self.llm_provider == LLMProvider.OPENAI and not self.openai_api_key:
+            raise ValueError("OpenAI provider requires OPENAI_API_KEY")
+        if self.llm_provider == LLMProvider.ANTHROPIC and not self.anthropic_api_key:
+            raise ValueError("Anthropic provider requires ANTHROPIC_API_KEY")
+        return self
+
     @property
     def is_production(self) -> bool:
         """Check if running in production mode."""
@@ -172,9 +187,11 @@ def get_settings() -> Settings:
 
     settings = Settings()
 
-    # Set OpenAI API key as environment variable for CrewAI compatibility
+    # Set API keys as environment variables for CrewAI compatibility
     # CrewAI checks environment variable even when LLM object is provided
     if settings.openai_api_key:
         os.environ["OPENAI_API_KEY"] = settings.openai_api_key
+    if settings.anthropic_api_key:
+        os.environ["ANTHROPIC_API_KEY"] = settings.anthropic_api_key
 
     return settings

domain/__init__.py

@@ -8,6 +8,8 @@
     Finding,
     FindingType,
     GroundTruthLabel,
+    Language,
+    LLMProvider,
     PRContext,
     PRMetadata,
     PRReviewResult,
@@ -24,11 +26,12 @@
     "Finding",
     "FindingType",
     "GroundTruthLabel",
+    "Language",
+    "LLMProvider",
     "PRContext",
     "PRMetadata",
     "PRReviewResult",
     "Severity",
     "SystemType",
     "ToolResult",
 ]
-