Skip to content

Commit 1276d2a

Browse files
committed
test(eval): add contract-edge graph fixture
Add a repo regression fixture where a trait-dispatched call reaches an unsafe SQL-building implementation, plus a checked-in loader test and TODO.md update completing the multi-hop graph fixture backlog item.
1 parent 7ebc62e commit 1276d2a

File tree

7 files changed

+82
-1
lines changed

7 files changed

+82
-1
lines changed

TODO.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ This roadmap is derived from deep research into Greptile's public docs, blog, MC
142142
91. [ ] Add eval fixtures for external-context alignment, not just diff-local correctness.
143143
92. [ ] Add eval fixtures for merge-readiness judgments and unresolved-blocker classification.
144144
93. [ ] Add eval fixtures for addressed-vs-stale finding lifecycle inference.
145-
94. [ ] Add eval fixtures for multi-hop graph reasoning across call chains and contract edges.
145+
94. [x] Add eval fixtures for multi-hop graph reasoning across call chains and contract edges.
146146
95. [ ] Add eval runs that compare single-pass review against agentic loop review.
147147
96. [ ] Add production replay evals using anonymized accepted/rejected review outcomes.
148148
97. [ ] Add leaderboard reporting for reviewer usefulness metrics, not just precision/recall.
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
use crate::search::QueryRunner;
2+
3+
pub struct PostgresQueryRunner;
4+
5+
impl QueryRunner for PostgresQueryRunner {
6+
fn find_user(&self, name: &str) -> String {
7+
format!("SELECT * FROM users WHERE name = '{}'", name)
8+
}
9+
}
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
pub struct Request {
2+
name: String,
3+
}
4+
5+
impl Request {
6+
pub fn name(&self) -> &str {
7+
&self.name
8+
}
9+
}
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
use crate::request::Request;
2+
use crate::search::QueryRunner;
3+
4+
pub fn get_profile(_runner: &dyn QueryRunner, _request: &Request) -> String {
5+
"ok".to_string()
6+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
pub trait QueryRunner {
2+
fn find_user(&self, name: &str) -> String;
3+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: repo regression - trait impl sql runner usage
2+
repo_path: graph_contract_edge_repo
3+
diff: |
4+
diff --git a/routes.rs b/routes.rs
5+
index 1111111..2222222 100644
6+
--- a/routes.rs
7+
+++ b/routes.rs
8+
@@ -2,4 +2,5 @@ use crate::request::Request;
9+
use crate::search::QueryRunner;
10+
11+
pub fn get_profile(runner: &dyn QueryRunner, request: &Request) -> String {
12+
- "ok".to_string()
13+
+ let user = runner.find_user(request.name());
14+
+ user
15+
}
16+
expect:
17+
must_find:
18+
- file: routes.rs
19+
contains_any:
20+
- sql injection
21+
- unsafe sql
22+
- trait implementation builds sql with user input
23+
- user-controlled input reaches the query runner implementation
24+
- interpolates user-controlled
25+
must_not_find:
26+
- contains: style
27+
min_total: 1
28+
max_total: 12

src/commands/eval/fixtures.rs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,32 @@ expect:
146146
);
147147
}
148148

149+
#[test]
150+
fn test_checked_in_repo_regression_contract_edge_fixture_loads_expected_repo_path() {
151+
let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
152+
.join("eval/fixtures/repo_regressions/trait_impl_sql_runner.yml");
153+
154+
let fixtures = load_eval_fixtures_from_path(&fixture_path).unwrap();
155+
156+
assert_eq!(fixtures.len(), 1);
157+
assert_eq!(
158+
fixtures[0].fixture.name.as_deref(),
159+
Some("repo regression - trait impl sql runner usage")
160+
);
161+
assert_eq!(
162+
fixtures[0].fixture.repo_path,
163+
Some(std::path::PathBuf::from("graph_contract_edge_repo"))
164+
);
165+
assert_eq!(
166+
fixtures[0].fixture.expect.must_find[0].file.as_deref(),
167+
Some("routes.rs")
168+
);
169+
assert!(fixtures[0].fixture.expect.must_find[0]
170+
.contains_any
171+
.iter()
172+
.any(|phrase| phrase.contains("trait implementation")));
173+
}
174+
149175
#[test]
150176
fn test_collect_eval_fixtures_expands_pack_entries_in_sorted_order() {
151177
let dir = tempdir().unwrap();

0 commit comments

Comments
 (0)