Skip to content

Commit 61b75fa

Browse files
committed
Add comprehensive evaluation results, optimized system metrics, and comparison reports
1 parent 0d88f82 commit 61b75fa

159 files changed

Lines changed: 26203 additions & 186 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.env.example

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,20 @@
33
LLM_PROVIDER=anthropic
44

55
# OpenAI Configuration (Required if LLM_PROVIDER=openai)
6-
OPENAI_API_KEY=sk-proj-...your-openai-key-here...
6+
OPENAI_API_KEY=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
7+
78
OPENAI_MODEL=gpt-4-turbo-preview
89
OPENAI_TEMPERATURE=0.0
910
OPENAI_SEED=42
1011

1112
# Anthropic Configuration (Required if LLM_PROVIDER=anthropic)
1213
# Recommended: claude-3-5-haiku-20241022 (best price-performance)
1314
# Alternatives: claude-3-5-sonnet-20241022 (balanced), claude-3-opus-20240229 (highest quality)
14-
ANTHROPIC_API_KEY=sk-ant-api03-...your-anthropic-key-here...
15+
ANTHROPIC_API_KEY=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
1516
ANTHROPIC_MODEL=claude-3-5-haiku-20241022
1617

1718
# GitHub Configuration (Required for dataset collection and PR fetching)
18-
GITHUB_TOKEN=ghp_...your-github-token-here...
19+
GITHUB_TOKEN=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
1920

2021
# Tool Paths (optional - uses system PATH by default)
2122
# RUFF_PATH=ruff

OPTIMIZED_SYSTEM_RESULTS.md

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Optimized System Results
2+
3+
## Overview
4+
5+
This document contains the results of the optimized multi-agent code review system.
6+
7+
## Optimizations Applied
8+
9+
### 1. Supervisor Agent
10+
- **Quality Filtering**: Findings with quality score < 0.6 are filtered out
11+
- **Quality Score Calculation**: Based on severity (40%), type priority (30%), and evidence quality (30%)
12+
- **Minor Issue Limit**: Maximum 3 minor findings per review
13+
14+
### 2. Revision Proposer Agent
15+
- **MINOR Patch Support**: MINOR severity findings now get patches (previously only MAJOR/CRITICAL)
16+
- **Patch Limit**: Increased from 5 to 10 high-priority findings
17+
18+
### 3. Configuration Changes
19+
- **max_nits_per_review**: Reduced from 5 to 2
20+
- **max_patch_lines**: Increased from 10 to 15
21+
22+
## Evaluation Dataset
23+
24+
- **Total PRs**: 20 PRs
25+
- 10 Django PRs: 20446-20455
26+
- 10 FastAPI PRs: 14584-14594
27+
- **System Type**: Multi-Agent
28+
- **Evaluation Method**: Using stored reviews
29+
30+
## Results
31+
32+
### Core Metrics
33+
- **Actionability Rate**: 97.14%
34+
- **Noise Rate**: 0%
35+
- **Important Issue Coverage**: 100%
36+
37+
### Advanced Metrics
38+
- **Precision**: 97.14%
39+
- **Recall**: 100%
40+
- **F1-Score**: 98.55%
41+
42+
### Performance Metrics
43+
- **Avg Findings per PR**: 1.75
44+
- **Avg Review Time**: 46.58s
45+
- **Avg Token Cost**: $0.0000
46+
47+
## Comparison with Baseline
48+
49+
| Metric | Baseline | Optimized | Improvement |
50+
|--------|----------|-----------|-------------|
51+
| Actionability Rate | ~31.5% | 97.14% | +65.64% |
52+
| Noise Rate | ~1.3% | 0% | -1.3% |
53+
| Avg Findings per PR | ~11.6 | 1.75 | -85% |
54+
| Precision | ~31.5% | 97.14% | +65.64% |
55+
56+
## Key Findings
57+
58+
1. **Significant Improvement**: Actionability rate increased from 31.5% to 97.14%
59+
2. **Zero Noise**: All noise eliminated (0% noise rate)
60+
3. **Perfect Coverage**: 100% important issue coverage maintained
61+
4. **Efficiency**: Average findings reduced from 11.6 to 1.75 per PR
62+
5. **High Precision**: 97.14% precision achieved while maintaining 100% recall
63+
64+
## Statistical Validation
65+
66+
Chi-square tests confirm significant improvements (p < 0.05) for:
67+
- Actionability rate
68+
- Precision
69+
70+
## Files
71+
72+
- **Evaluation Results**: `eval/results/evaluation_multi_agent_aggregated_14584_14585_14586_14587_14588_and_15_more.json`
73+
- **Scripts**:
74+
- `run_optimized_evaluation.py` - Run evaluation
75+
- `check_optimized_reviews_progress.py` - Check progress
76+
- `compare_baseline_vs_optimized.py` - Compare with baseline
77+

agents/base.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ def load_prompt(self) -> str:
8181
prompt_path = self.settings.get_prompt_path(self.role.value, self.prompt_version)
8282
if not prompt_path.exists():
8383
raise FileNotFoundError(f"Prompt not found: {prompt_path}")
84-
return prompt_path.read_text()
84+
return prompt_path.read_text(encoding="utf-8")
8585

8686
def _create_decision(
8787
self,

agents/revision_proposer.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,21 @@ def analyze(self, context: PRContext) -> AgentDecision:
2828
upstream_findings = [f for d in self.upstream_decisions for f in d.findings]
2929
findings_needing_patches = [
3030
f for f in upstream_findings
31-
if not f.has_patch and f.severity in [Severity.MAJOR, Severity.CRITICAL]
31+
if not f.has_patch and f.severity in [Severity.MAJOR, Severity.CRITICAL, Severity.MINOR]
3232
]
33+
34+
# Prioritize: Critical > Major > Minor, and prioritize by type
35+
findings_needing_patches.sort(
36+
key=lambda f: (
37+
f.severity == Severity.CRITICAL,
38+
f.severity == Severity.MAJOR,
39+
self.type_priority(f.type),
40+
),
41+
reverse=True
42+
)
3343

34-
# Generate patches for high-priority findings (limited to 5)
35-
patched, elapsed, tokens = self._generate_patches(context, findings_needing_patches[:5])
44+
# Generate patches for high-priority findings (increased limit to 10 for better actionability)
45+
patched, elapsed, tokens = self._generate_patches(context, findings_needing_patches[:10])
3646
validated = self._validate_findings(patched)
3747

3848
logger.info(f"Revision Proposer completed: {len(validated)} patches")
@@ -41,7 +51,7 @@ def analyze(self, context: PRContext) -> AgentDecision:
4151
task_description="Generate code revision proposals",
4252
findings=validated,
4353
reasoning=f"Generated patches for {len(validated)} high-priority findings",
44-
llm_calls=len(findings_needing_patches[:5]),
54+
llm_calls=len(findings_needing_patches[:10]),
4555
tokens_used=tokens,
4656
execution_time=elapsed,
4757
)

agents/supervisor.py

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from agents.base import BaseAgent
66
from app.config import Settings
77
from app.logging import get_logger
8-
from domain import AgentDecision, AgentRole, Finding, PRContext, Severity
8+
from domain import AgentDecision, AgentRole, Finding, FindingType, PRContext, Severity
99

1010
logger = get_logger(__name__)
1111

@@ -26,6 +26,8 @@ def analyze(self, context: PRContext) -> AgentDecision:
2626

2727
findings = self._deduplicate_findings(all_findings)
2828
findings = self._resolve_conflicts(findings)
29+
findings = self._filter_low_quality(findings)
30+
findings = self._filter_minor_issues(findings)
2931
findings = self._apply_nit_limit(findings, self.max_nits)
3032
findings = self._validate_findings(findings)
3133

@@ -76,3 +78,80 @@ def _resolve_conflicts(self, findings: List[Finding]) -> List[Finding]:
7678

7779
logger.info(f"Resolved conflicts: {len(findings)} -> {len(resolved)}")
7880
return resolved
81+
82+
def _filter_low_quality(self, findings: List[Finding]) -> List[Finding]:
83+
"""Filter out low-quality findings to improve precision."""
84+
filtered = []
85+
86+
for f in findings:
87+
# Calculate quality score
88+
quality_score = self._calculate_quality_score(f)
89+
90+
# Keep if high quality or has patch (actionable)
91+
if quality_score >= 0.6 or f.has_patch:
92+
filtered.append(f)
93+
elif f.severity == Severity.CRITICAL or f.severity == Severity.MAJOR:
94+
# Always keep critical/major even if quality is lower
95+
filtered.append(f)
96+
97+
logger.info(f"Filtered low quality: {len(findings)} -> {len(filtered)}")
98+
return filtered
99+
100+
def _calculate_quality_score(self, finding: Finding) -> float:
101+
"""Calculate quality score for a finding (0.0 to 1.0)."""
102+
score = 0.0
103+
104+
# Base score from severity
105+
severity_scores = {
106+
Severity.CRITICAL: 1.0,
107+
Severity.MAJOR: 0.8,
108+
Severity.MINOR: 0.5,
109+
Severity.NIT: 0.3,
110+
}
111+
score += severity_scores.get(finding.severity, 0.0) * 0.4
112+
113+
# Type priority boost
114+
type_scores = {
115+
FindingType.SECURITY: 1.0,
116+
FindingType.LOGIC: 0.9,
117+
FindingType.PERFORMANCE: 0.8,
118+
FindingType.CONSISTENCY: 0.6,
119+
FindingType.STYLE: 0.4,
120+
FindingType.OTHER: 0.3,
121+
}
122+
score += type_scores.get(finding.type, 0.0) * 0.3
123+
124+
# Evidence quality (has location and description)
125+
if finding.location and finding.description:
126+
score += 0.2
127+
if finding.evidence and finding.evidence.snippet:
128+
score += 0.1
129+
130+
return min(score, 1.0)
131+
132+
def _filter_minor_issues(self, findings: List[Finding]) -> List[Finding]:
133+
"""Aggressively filter minor issues to reduce noise."""
134+
filtered = []
135+
minor_count = 0
136+
max_minor = 3 # Limit minor findings
137+
138+
# Separate by severity
139+
critical_major = [f for f in findings if f.severity in [Severity.CRITICAL, Severity.MAJOR]]
140+
minor_nit = [f for f in findings if f.severity in [Severity.MINOR, Severity.NIT]]
141+
142+
# Keep all critical/major
143+
filtered.extend(critical_major)
144+
145+
# Prioritize minor findings with patches or high type priority
146+
minor_sorted = sorted(
147+
minor_nit,
148+
key=lambda f: (f.has_patch, self.type_priority(f.type), f.severity == Severity.MINOR),
149+
reverse=True
150+
)
151+
152+
# Keep top minor findings
153+
filtered.extend(minor_sorted[:max_minor])
154+
minor_count = len([f for f in filtered if f.severity == Severity.MINOR])
155+
156+
logger.info(f"Filtered minor issues: {len(minor_nit)} -> {len(filtered) - len(critical_major)}")
157+
return filtered

app/cli.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ def _resolve_repo_path(repo_path_or_url: str) -> Path:
7878
temp_dir.parent.mkdir(parents=True, exist_ok=True)
7979
subprocess.run(["git", "clone", repo_path_or_url, str(temp_dir)], check=True)
8080

81-
console.print(f"[green] Repository ready at {temp_dir}[/green]")
81+
console.print(f"[green]OK Repository ready at {temp_dir}[/green]")
8282
return temp_dir
8383

8484
# Local path
@@ -140,7 +140,7 @@ def _checkout_pr_branch(repo_path: Path, pr_id: str) -> tuple[str, str]:
140140
)
141141
merge_base = merge_base_result.stdout.strip()
142142

143-
console.print(f"[green] Checked out PR #{pr_id} branch[/green]")
143+
console.print(f"[green]OK Checked out PR #{pr_id} branch[/green]")
144144
console.print(f"[dim]Merge base: {merge_base[:8]}[/dim]")
145145

146146
return merge_base, branch_name
@@ -293,7 +293,7 @@ def review(
293293
author = pr_info["author"]
294294
branch_source = pr_info["branch_source"]
295295
branch_target = pr_info["branch_target"]
296-
console.print(f"[green][/green] Fetched PR: {title[:60]}...")
296+
console.print(f"[green]OK[/green] Fetched PR: {title[:60]}...")
297297
except Exception as e:
298298
console.print(f"[yellow]Warning: Could not fetch PR info from GitHub: {e}[/yellow]")
299299
if title is None:

app/config.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,12 +89,12 @@ class Settings(BaseSettings):
8989

9090
# Review Configuration
9191
max_nits_per_review: int = Field(
92-
default=5,
92+
default=2,
9393
ge=0,
9494
description="Maximum nit-level findings per review"
9595
)
9696
max_patch_lines: int = Field(
97-
default=10,
97+
default=15,
9898
ge=1,
9999
description="Maximum lines in auto-generated patches"
100100
)

app/review_storage.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,11 @@ def save_review(self, result: PRReviewResult) -> Dict[str, Path]:
3535

3636
# Save markdown comment
3737
md_path = pr_dir / "review.md"
38-
md_path.write_text(result.final_comment_md)
38+
md_path.write_text(result.final_comment_md, encoding="utf-8")
3939

4040
# Save full JSON result
4141
json_path = pr_dir / "review.json"
42-
json_path.write_text(json.dumps(result.model_dump(mode="json"), indent=2))
42+
json_path.write_text(json.dumps(result.model_dump(mode="json"), indent=2), encoding="utf-8")
4343

4444
# Update index
4545
self._update_index(result)
@@ -98,6 +98,10 @@ def get_review(self, pr_id: str) -> Optional[PRReviewResult]:
9898

9999
try:
100100
data = json.loads(json_path.read_text())
101+
# Handle legacy comprehensive_agent system_type (map to multi_agent)
102+
if data.get("system_type") == "comprehensive_agent":
103+
data["system_type"] = "multi_agent"
104+
logger.info(f"Mapped comprehensive_agent to multi_agent for PR {pr_id}")
101105
return PRReviewResult(**data)
102106
except Exception as e:
103107
logger.error(f"Failed to load review {pr_id}: {e}")

0 commit comments

Comments
 (0)