Skip to content

Commit 7c6a38c

Browse files
test: add code review fixtures and pytest coverage
1 parent 4dba7dc commit 7c6a38c

10 files changed

Lines changed: 204 additions & 1 deletion

File tree

examples/skills_code_review_agent/README.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,27 @@ Using the `sqlite3` CLI:
5757
sqlite3 examples/skills_code_review_agent/output/reviews.sqlite3 "select severity, category, file, line, title from findings order by id;"
5858
```
5959

60+
## Run Tests
61+
62+
```bash
63+
python -m pytest examples/skills_code_review_agent/tests
64+
```
65+
66+
The tests run only local parsing, rules, redaction, report generation, and
67+
dedupe checks. They do not call an LLM, Docker, Cube, remote network, or any
68+
external service.
69+
70+
## Fixtures
71+
72+
- `clean.diff`: safe changes; should produce no high-severity findings.
73+
- `security.diff`: mixed security sample covering secret, missing timeout, broad exception, SQL risk, and resource lifecycle.
74+
- `sql_injection.diff`: SQL string concatenation.
75+
- `missing_timeout.diff`: `httpx` request without `timeout=`.
76+
- `broad_except.diff`: broad `except Exception` with swallowed failure.
77+
- `resource_leak.diff`: `open(...)` without a context manager.
78+
- `duplicate.diff`: duplicate hunk producing the same finding, used to verify dedupe.
79+
- `secret_redaction.diff`: multiple secret-like values, used to verify reports omit plaintext secrets.
80+
6081
## Current Scope
6182

6283
- Parses unified diff hunks and added line numbers.
@@ -69,6 +90,7 @@ sqlite3 examples/skills_code_review_agent/output/reviews.sqlite3 "select severit
6990
- Redacts likely API keys, tokens, secrets, and passwords before writing reports.
7091
- Produces `review_report.json` and `review_report.md`.
7192
- Optionally persists review tasks, findings, and report metadata to SQLite.
93+
- Includes local pytest coverage for the deterministic rule fixtures.
7294

7395
## Not Implemented Yet
7496

examples/skills_code_review_agent/agent/rules.py

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,27 @@ def _next_added_line(file_lines: list[ChangedLine], index: int) -> ChangedLine |
110110
return file_lines[index + 1]
111111

112112

113+
def _dedupe_findings(findings: list[Finding]) -> list[Finding]:
114+
seen: set[tuple[object, ...]] = set()
115+
deduped: list[Finding] = []
116+
for finding in findings:
117+
key = (
118+
finding.severity,
119+
finding.category,
120+
finding.file,
121+
finding.line,
122+
finding.title,
123+
finding.evidence,
124+
finding.recommendation,
125+
finding.source,
126+
)
127+
if key in seen:
128+
continue
129+
seen.add(key)
130+
deduped.append(finding)
131+
return deduped
132+
133+
113134
def run_static_rules(changed_lines: Iterable[ChangedLine]) -> list[Finding]:
114135
"""Run all deterministic phase-1 rules over added diff lines."""
115136

@@ -192,4 +213,4 @@ def run_static_rules(changed_lines: Iterable[ChangedLine]) -> list[Finding]:
192213
)
193214
)
194215

195-
return sorted(findings, key=lambda item: (item.file, item.line, item.category, item.source))
216+
return sorted(_dedupe_findings(findings), key=lambda item: (item.file, item.line, item.category, item.source))
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
diff --git a/app/worker.py b/app/worker.py
2+
index 3000001..3000002 100644
3+
--- a/app/worker.py
4+
+++ b/app/worker.py
5+
@@ -1,5 +1,10 @@
6+
def run_job(job):
7+
- return job.run()
8+
+ try:
9+
+ return job.run()
10+
+ except Exception:
11+
+ return None
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
diff --git a/app/duplicate.py b/app/duplicate.py
2+
index 5000001..5000002 100644
3+
--- a/app/duplicate.py
4+
+++ b/app/duplicate.py
5+
@@ -1,3 +1,4 @@
6+
import requests
7+
def fetch():
8+
+ return requests.get("https://duplicate.example/api")
9+
@@ -1,3 +1,4 @@
10+
import requests
11+
def fetch():
12+
+ return requests.get("https://duplicate.example/api")
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
diff --git a/app/client.py b/app/client.py
2+
index 2000001..2000002 100644
3+
--- a/app/client.py
4+
+++ b/app/client.py
5+
@@ -1,5 +1,7 @@
6+
import httpx
7+
8+
9+
def load_status():
10+
- return {"ok": True}
11+
+ return httpx.get("https://status.example/api")
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
diff --git a/app/files.py b/app/files.py
2+
index 4000001..4000002 100644
3+
--- a/app/files.py
4+
+++ b/app/files.py
5+
@@ -1,4 +1,6 @@
6+
def read_token(path):
7+
- return ""
8+
+ handle = open(path, encoding="utf-8")
9+
+ return handle.read()
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
diff --git a/app/settings.py b/app/settings.py
2+
index 6000001..6000002 100644
3+
--- a/app/settings.py
4+
+++ b/app/settings.py
5+
@@ -1,4 +1,8 @@
6+
def settings():
7+
- return {}
8+
+ api_key = "sk-test-abcdefghijklmnop"
9+
+ auth_token = "token-value-1234567890"
10+
+ password = "correct-horse-prod-password"
11+
+ return {"enabled": True}
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
diff --git a/app/search.py b/app/search.py
2+
index 1000001..1000002 100644
3+
--- a/app/search.py
4+
+++ b/app/search.py
5+
@@ -1,4 +1,6 @@
6+
def find_user(db, user_id):
7+
- return db.execute("SELECT * FROM users WHERE id = :id", {"id": user_id})
8+
+ sql = "SELECT * FROM users WHERE id = " + user_id
9+
+ return db.execute(sql)
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
"""Pytest setup for the self-contained code review example."""
2+
3+
from __future__ import annotations
4+
5+
import sys
6+
from pathlib import Path
7+
8+
9+
EXAMPLE_ROOT = Path(__file__).resolve().parents[1]
10+
if str(EXAMPLE_ROOT) not in sys.path:
11+
sys.path.insert(0, str(EXAMPLE_ROOT))
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
"""Deterministic tests for the phase-1/2 code review example."""
2+
3+
from __future__ import annotations
4+
5+
from pathlib import Path
6+
7+
from agent.diff_parser import ParsedDiff
8+
from agent.diff_parser import parse_unified_diff
9+
from agent.findings import Finding
10+
from agent.report import build_report
11+
from agent.report import write_reports
12+
from agent.rules import run_static_rules
13+
14+
15+
EXAMPLE_ROOT = Path(__file__).resolve().parents[1]
16+
FIXTURES = EXAMPLE_ROOT / "fixtures"
17+
18+
19+
def _review_fixture(name: str) -> tuple[ParsedDiff, list[Finding]]:
20+
diff_text = (FIXTURES / name).read_text(encoding="utf-8")
21+
parsed = parse_unified_diff(diff_text)
22+
findings = run_static_rules(parsed.changed_lines)
23+
return parsed, findings
24+
25+
26+
def _categories(findings: list[Finding]) -> set[str]:
27+
return {finding.category for finding in findings}
28+
29+
30+
def _count_category(findings: list[Finding], category: str) -> int:
31+
return sum(1 for finding in findings if finding.category == category)
32+
33+
34+
def test_clean_diff_has_no_high_findings() -> None:
35+
_, findings = _review_fixture("clean.diff")
36+
assert [finding for finding in findings if finding.severity == "high"] == []
37+
38+
39+
def test_security_diff_contains_expected_categories() -> None:
40+
_, findings = _review_fixture("security.diff")
41+
categories = _categories(findings)
42+
assert "secret" in categories
43+
assert "network-timeout" in categories
44+
assert "error-handling" in categories
45+
46+
47+
def test_sql_injection_fixture() -> None:
48+
_, findings = _review_fixture("sql_injection.diff")
49+
assert "sql-injection" in _categories(findings)
50+
51+
52+
def test_missing_timeout_fixture() -> None:
53+
_, findings = _review_fixture("missing_timeout.diff")
54+
assert "network-timeout" in _categories(findings)
55+
56+
57+
def test_broad_except_fixture() -> None:
58+
_, findings = _review_fixture("broad_except.diff")
59+
assert "error-handling" in _categories(findings)
60+
61+
62+
def test_resource_leak_fixture() -> None:
63+
_, findings = _review_fixture("resource_leak.diff")
64+
assert "resource-lifecycle" in _categories(findings)
65+
66+
67+
def test_secret_redaction_report_omits_plaintext_values(tmp_path: Path) -> None:
68+
parsed, findings = _review_fixture("secret_redaction.diff")
69+
report = build_report(
70+
diff_file="fixtures/secret_redaction.diff",
71+
files=parsed.files,
72+
findings=findings,
73+
dry_run=True,
74+
)
75+
json_path, md_path = write_reports(report, tmp_path)
76+
combined = json_path.read_text(encoding="utf-8") + md_path.read_text(encoding="utf-8")
77+
78+
assert "sk-test-abcdefghijklmnop" not in combined
79+
assert "token-value-1234567890" not in combined
80+
assert "correct-horse-prod-password" not in combined
81+
assert "<redacted:" in combined
82+
83+
84+
def test_duplicate_fixture_dedupes_repeated_finding() -> None:
85+
_, findings = _review_fixture("duplicate.diff")
86+
assert _count_category(findings, "network-timeout") == 1

0 commit comments

Comments
 (0)