Skip to content

Commit fdc2c6c

Browse files
author
Hakan Bogan
committed
improved metrics
1 parent d83331a commit fdc2c6c

21 files changed

Lines changed: 1692 additions & 1216 deletions

‎README.md‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -191,15 +191,22 @@ See [eval/dataset/README.md](eval/dataset/README.md) for detailed instructions.
191191
Run evaluation on collected dataset:
192192

193193
```bash
194-
# Single-agent baseline
194+
# Evaluate using stored reviews (recommended)
195195
poetry run python -m app.cli evaluate \
196-
--system single_agent \
197-
/path/to/repo
196+
--system multi_agent \
197+
--use-stored
198198

199-
# Multi-agent (proposed)
199+
# Evaluate specific PRs
200200
poetry run python -m app.cli evaluate \
201201
--system multi_agent \
202-
/path/to/repo
202+
--pr-ids "14468,2779" \
203+
--use-stored
204+
205+
# Re-run reviews and evaluate
206+
poetry run python -m app.cli evaluate \
207+
--system single_agent \
208+
--rerun \
209+
--repo-path /path/to/repo
203210

204211
# Compare systems
205212
poetry run python -m app.cli compare \

‎domain/models.py‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -224,20 +224,6 @@ class EvaluationResult(BaseModel):
224224
avg_review_time_s: float = Field(..., description="Average review time")
225225
avg_token_cost: float = Field(..., description="Average token cost in USD")
226226

227-
# CTR/CL/SI scores (thesis-specific)
228-
change_type_recognition_score: float = Field(
229-
..., ge=0.0, le=1.0,
230-
description="Accuracy in understanding change type"
231-
)
232-
change_location_score: float = Field(
233-
..., ge=0.0, le=1.0,
234-
description="Accuracy in pinpointing issues"
235-
)
236-
solution_logic_score: float = Field(
237-
..., ge=0.0, le=1.0,
238-
description="Quality of proposed solutions"
239-
)
240-
241227
# Statistical data
242228
confidence_interval_95: Dict[str, tuple[float, float]] = Field(
243229
default_factory=dict,

‎eval/dataset/categorized/bugfix/ground_truth.json‎

Lines changed: 110 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,125 @@
11
[
22
{
3-
"pr_id": "20273",
4-
"important_issues": [],
5-
"false_positive_tolerance": 0,
6-
"labeler_id": "github_reviewers",
7-
"labeled_at": "2025-11-30T12:44:49.315322",
8-
"notes": "Extracted from 1 review comments"
3+
"pr_id": "dummy-bugfix-0012",
4+
"important_issues": [
5+
"Need to verify fix doesn't break existing behavior"
6+
],
7+
"false_positive_tolerance": 1,
8+
"labeler_id": "synthetic_expert_review",
9+
"labeled_at": "2025-12-07T18:37:25.737460",
10+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
11+
},
12+
{
13+
"pr_id": "dummy-bugfix-0034",
14+
"important_issues": [
15+
"Error handling incomplete"
16+
],
17+
"false_positive_tolerance": 2,
18+
"labeler_id": "synthetic_expert_review",
19+
"labeled_at": "2025-12-07T18:37:25.737574",
20+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
21+
},
22+
{
23+
"pr_id": "dummy-bugfix-0037",
24+
"important_issues": [
25+
"Missing regression test for the bug fix",
26+
"Error handling incomplete",
27+
"Edge case not handled in fix"
28+
],
29+
"false_positive_tolerance": 2,
30+
"labeler_id": "synthetic_expert_review",
31+
"labeled_at": "2025-12-07T18:37:25.737583",
32+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
33+
},
34+
{
35+
"pr_id": "dummy-bugfix-0058",
36+
"important_issues": [
37+
"Need to verify fix doesn't break existing behavior",
38+
"Edge case not handled in fix",
39+
"Missing regression test for the bug fix"
40+
],
41+
"false_positive_tolerance": 1,
42+
"labeler_id": "synthetic_expert_review",
43+
"labeled_at": "2025-12-07T18:37:25.737589",
44+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
945
},
1046
{
11-
"pr_id": "62699",
47+
"pr_id": "dummy-bugfix-0065",
1248
"important_issues": [
13-
"since this was a bug in development code, you don't need to update whatsnew... @ doc/source/whatsnew/v3.0.0.rst:1099"
49+
"Edge case not handled in fix",
50+
"Fix may introduce new bug in related code"
1451
],
1552
"false_positive_tolerance": 2,
16-
"labeler_id": "github_reviewers",
17-
"labeled_at": "2025-11-30T12:55:20.078731",
18-
"notes": "Extracted from 8 review comments"
53+
"labeler_id": "synthetic_expert_review",
54+
"labeled_at": "2025-12-07T18:37:25.737594",
55+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
1956
},
2057
{
21-
"pr_id": "61920",
58+
"pr_id": "dummy-bugfix-0076",
2259
"important_issues": [
23-
"The reason I kept this is because [this test](https://github.com/pandas-dev/pandas/blob/e72c8a1e0ad4... @ pandas/core/arrays/interval.py:1995",
24-
"removing ravel here causes a failure in the _shallow_copy() call, with the error: “Index data must b... @ pandas/core/arrays/interval.py:1992",
25-
"@rhshadrach I added this check to handle mypy failure. should I add a separate test for this one?... @ pandas/core/arrays/interval.py:2139"
60+
"Edge case not handled in fix"
2661
],
27-
"false_positive_tolerance": 5,
28-
"labeler_id": "github_reviewers",
29-
"labeled_at": "2025-11-30T12:55:20.080375",
30-
"notes": "Extracted from 25 review comments"
62+
"false_positive_tolerance": 2,
63+
"labeler_id": "synthetic_expert_review",
64+
"labeled_at": "2025-12-07T18:37:25.737599",
65+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
3166
},
3267
{
33-
"pr_id": "32592",
34-
"important_issues": [],
35-
"false_positive_tolerance": 0,
36-
"labeler_id": "github_reviewers",
37-
"labeled_at": "2025-11-30T12:58:11.647353",
38-
"notes": "Extracted from 1 review comments"
68+
"pr_id": "dummy-bugfix-0077",
69+
"important_issues": [
70+
"Fix may introduce new bug in related code",
71+
"Error handling incomplete"
72+
],
73+
"false_positive_tolerance": 2,
74+
"labeler_id": "synthetic_expert_review",
75+
"labeled_at": "2025-12-07T18:37:25.737604",
76+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
77+
},
78+
{
79+
"pr_id": "dummy-bugfix-0080",
80+
"important_issues": [
81+
"Fix may introduce new bug in related code",
82+
"Need to verify fix doesn't break existing behavior",
83+
"Error handling incomplete"
84+
],
85+
"false_positive_tolerance": 1,
86+
"labeler_id": "synthetic_expert_review",
87+
"labeled_at": "2025-12-07T18:37:25.737609",
88+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
89+
},
90+
{
91+
"pr_id": "dummy-bugfix-0081",
92+
"important_issues": [
93+
"Edge case not handled in fix",
94+
"Need to verify fix doesn't break existing behavior",
95+
"Error handling incomplete",
96+
"Missing regression test for the bug fix"
97+
],
98+
"false_positive_tolerance": 2,
99+
"labeler_id": "synthetic_expert_review",
100+
"labeled_at": "2025-12-07T18:37:25.737615",
101+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
102+
},
103+
{
104+
"pr_id": "dummy-bugfix-0095",
105+
"important_issues": [
106+
"Need to verify fix doesn't break existing behavior"
107+
],
108+
"false_positive_tolerance": 2,
109+
"labeler_id": "synthetic_expert_review",
110+
"labeled_at": "2025-12-07T18:37:25.737619",
111+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
112+
},
113+
{
114+
"pr_id": "dummy-bugfix-0096",
115+
"important_issues": [
116+
"Fix may introduce new bug in related code",
117+
"Error handling incomplete",
118+
"Missing regression test for the bug fix"
119+
],
120+
"false_positive_tolerance": 2,
121+
"labeler_id": "synthetic_expert_review",
122+
"labeled_at": "2025-12-07T18:37:25.737624",
123+
"notes": "Synthetic ground truth for bugfix PR based on common review patterns"
39124
}
40125
]

‎eval/dataset/categorized/bugfix/metadata.json‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11
{
22
"category": "bugfix",
33
"pr_count": 100,
4-
"ground_truth_count": 30,
4+
"ground_truth_count": 11,
55
"statistics": {
66
"avg_lines_changed": 136.4,
77
"avg_complexity_score": 25.0,
8-
"total_important_issues": 64,
9-
"avg_issues_per_pr": 2.1
8+
"total_important_issues": 24,
9+
"avg_issues_per_pr": 2.2
1010
},
1111
"repositories": [
12-
"scikit-learn/scikit-learn",
13-
"pandas-dev/pandas",
14-
"django/django",
12+
"flask/flask",
1513
"requests/requests",
16-
"tensorflow/tensorflow",
14+
"django/django",
15+
"fastapi/fastapi",
16+
"pandas-dev/pandas",
17+
"scikit-learn/scikit-learn",
18+
"pytorch/pytorch",
1719
"numpy/numpy",
1820
"scipy/scipy",
19-
"fastapi/fastapi",
20-
"flask/flask",
21-
"pytorch/pytorch"
21+
"tensorflow/tensorflow"
2222
]
2323
}

‎eval/dataset/categorized/feature/ground_truth.json‎

Lines changed: 70 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,79 @@
11
[
22
{
3-
"pr_id": "62641",
4-
"important_issues": [],
3+
"pr_id": "dummy-feature-0005",
4+
"important_issues": [
5+
"Test coverage insufficient for new feature",
6+
"Missing validation for user inputs",
7+
"Backward compatibility not addressed",
8+
"Performance impact not evaluated",
9+
"API design needs review - parameter naming unclear",
10+
"Missing documentation for new feature"
11+
],
12+
"false_positive_tolerance": 3,
13+
"labeler_id": "synthetic_expert_review",
14+
"labeled_at": "2025-12-07T18:37:25.743231",
15+
"notes": "Synthetic ground truth for feature PR based on common review patterns"
16+
},
17+
{
18+
"pr_id": "dummy-feature-0048",
19+
"important_issues": [
20+
"Test coverage insufficient for new feature",
21+
"Missing validation for user inputs"
22+
],
23+
"false_positive_tolerance": 2,
24+
"labeler_id": "synthetic_expert_review",
25+
"labeled_at": "2025-12-07T18:37:25.743253",
26+
"notes": "Synthetic ground truth for feature PR based on common review patterns"
27+
},
28+
{
29+
"pr_id": "dummy-feature-0053",
30+
"important_issues": [
31+
"API design needs review - parameter naming unclear",
32+
"Missing documentation for new feature",
33+
"Backward compatibility not addressed",
34+
"Missing validation for user inputs"
35+
],
36+
"false_positive_tolerance": 3,
37+
"labeler_id": "synthetic_expert_review",
38+
"labeled_at": "2025-12-07T18:37:25.743259",
39+
"notes": "Synthetic ground truth for feature PR based on common review patterns"
40+
},
41+
{
42+
"pr_id": "dummy-feature-0058",
43+
"important_issues": [
44+
"Backward compatibility not addressed",
45+
"Test coverage insufficient for new feature",
46+
"Missing validation for user inputs",
47+
"Performance impact not evaluated",
48+
"API design needs review - parameter naming unclear"
49+
],
550
"false_positive_tolerance": 2,
6-
"labeler_id": "github_reviewers",
7-
"labeled_at": "2025-11-30T12:55:20.081355",
8-
"notes": "Extracted from 6 review comments"
51+
"labeler_id": "synthetic_expert_review",
52+
"labeled_at": "2025-12-07T18:37:25.743266",
53+
"notes": "Synthetic ground truth for feature PR based on common review patterns"
954
},
1055
{
11-
"pr_id": "32582",
56+
"pr_id": "dummy-feature-0062",
1257
"important_issues": [
13-
"Yes, you are right that only NumPy supports string arrays.\r\n\r\nHere, I'm considering the case where `... @ sklearn/preprocessing/_label.py:574",
14-
"I don't understand in which context one would expect integer valued `y` and `str` valued `classes`. ... @ sklearn/preprocessing/tests/test_label.py:751",
15-
"I realised that we have to convert `classes` into an array anyway, since the `_isin` function doesn’... @ sklearn/preprocessing/_label.py:588",
16-
"Suggestion to remove the additional newline before except\r\n```suggestion\r\n try:\r\n classes ... @ sklearn/preprocessing/_label.py:584",
17-
"```suggestion\r\n # Numeric class labels should not raise any errors for non-NumPy namespac... @ sklearn/preprocessing/tests/test_label.py:791",
18-
"For the helper functions `_inverse_binarize_multiclass` and `_inverse_binarize_thresholding`, the `c... @ sklearn/preprocessing/_label.py:748",
19-
"Do you think it would be informative to add `from e` here? (e.g., would the original error say what ... @ sklearn/preprocessing/_label.py:588"
58+
"Backward compatibility not addressed",
59+
"API design needs review - parameter naming unclear",
60+
"Missing documentation for new feature"
2061
],
21-
"false_positive_tolerance": 5,
22-
"labeler_id": "github_reviewers",
23-
"labeled_at": "2025-11-30T12:58:11.649506",
24-
"notes": "Extracted from 30 review comments"
62+
"false_positive_tolerance": 3,
63+
"labeler_id": "synthetic_expert_review",
64+
"labeled_at": "2025-12-07T18:37:25.743271",
65+
"notes": "Synthetic ground truth for feature PR based on common review patterns"
66+
},
67+
{
68+
"pr_id": "dummy-feature-0100",
69+
"important_issues": [
70+
"API design needs review - parameter naming unclear",
71+
"Missing documentation for new feature",
72+
"Missing validation for user inputs"
73+
],
74+
"false_positive_tolerance": 2,
75+
"labeler_id": "synthetic_expert_review",
76+
"labeled_at": "2025-12-07T18:37:25.743278",
77+
"notes": "Synthetic ground truth for feature PR based on common review patterns"
2578
}
2679
]

‎eval/dataset/categorized/feature/metadata.json‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11
{
22
"category": "feature",
33
"pr_count": 100,
4-
"ground_truth_count": 26,
4+
"ground_truth_count": 6,
55
"statistics": {
6-
"avg_lines_changed": 366.3,
7-
"avg_complexity_score": 38.1,
8-
"total_important_issues": 83,
9-
"avg_issues_per_pr": 3.2
6+
"avg_lines_changed": 370.5,
7+
"avg_complexity_score": 38.5,
8+
"total_important_issues": 23,
9+
"avg_issues_per_pr": 3.8
1010
},
1111
"repositories": [
12+
"flask/flask",
13+
"requests/requests",
1214
"django/django",
13-
"scikit-learn/scikit-learn",
15+
"fastapi/fastapi",
1416
"pandas-dev/pandas",
15-
"requests/requests",
16-
"tensorflow/tensorflow",
17+
"scikit-learn/scikit-learn",
18+
"pytorch/pytorch",
1719
"numpy/numpy",
1820
"scipy/scipy",
19-
"fastapi/fastapi",
20-
"flask/flask",
21-
"pytorch/pytorch"
21+
"tensorflow/tensorflow"
2222
]
2323
}

0 commit comments

Comments
 (0)