Skip to content

Commit e0b50c6

Browse files
haasonsaashaasonsaas
committed
feat(eval): add infra misconfiguration fixtures
Add a review-depth infrastructure benchmark pack for Docker, Terraform, GitHub Actions, and Kubernetes risks, plus a checked-in pack loader test.
1 parent 294c3c2 commit e0b50c6

File tree

2 files changed

+200
-0
lines changed

2 files changed

+200
-0
lines changed

‎eval/fixtures/deep_review_suite/review_depth_infra.json‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
{
2+
"name": "review-depth-infra",
3+
"author": "diffscope",
4+
"version": "1.0.0",
5+
"description": "Infrastructure and configuration security benchmark pack for deeper live review runs.",
6+
"languages": [
7+
"docker",
8+
"hcl",
9+
"yaml"
10+
],
11+
"categories": [
12+
"security"
13+
],
14+
"thresholds": {
15+
"min_precision": 0.45,
16+
"min_recall": 0.35,
17+
"min_f1": 0.4,
18+
"max_false_positive_rate": 0.45,
19+
"min_weighted_score": 0.45
20+
},
21+
"metadata": {
22+
"purpose": "live-regression-eval",
23+
"family": "review-depth"
24+
},
25+
"fixtures": [
26+
{
27+
"name": "docker-user-root",
28+
"category": "security",
29+
"language": "docker",
30+
"difficulty": "Easy",
31+
"diff_content": "diff --git a/Dockerfile b/Dockerfile\nindex 1111111..2222222 100644\n--- a/Dockerfile\n+++ b/Dockerfile\n@@ -1,5 +1,5 @@\n FROM node:20-alpine\n RUN adduser -D appuser\n WORKDIR /app\n-USER appuser\n+USER root\n CMD [\"node\", \"server.js\"]\n",
32+
"expected_findings": [
33+
{
34+
"description": "Container is explicitly configured to run as root instead of a non-root user.",
35+
"severity": "Warning",
36+
"category": "Security",
37+
"file_pattern": "Dockerfile",
38+
"line_hint": 4,
39+
"contains_any": [
40+
"container runs as root",
41+
"user root",
42+
"should run as a non-root user",
43+
"drop privileges",
44+
"least privilege"
45+
],
46+
"rule_id": "sec.infra.docker-root"
47+
}
48+
],
49+
"negative_findings": [
50+
{
51+
"description": "Avoid style-only comments.",
52+
"contains": "style"
53+
}
54+
],
55+
"min_total": 1,
56+
"max_total": 8,
57+
"description": "Containers should not be switched back to root for runtime execution.",
58+
"source": "deep-review-suite"
59+
},
60+
{
61+
"name": "terraform-ssh-open-world",
62+
"category": "security",
63+
"language": "hcl",
64+
"difficulty": "Medium",
65+
"diff_content": "diff --git a/main.tf b/main.tf\nindex 1111111..2222222 100644\n--- a/main.tf\n+++ b/main.tf\n@@ -2,7 +2,7 @@ resource \"aws_security_group\" \"ssh\" {\n name = \"public-ssh\"\n from_port = 22\n to_port = 22\n protocol = \"tcp\"\n- cidr_blocks = [\"10.0.0.0/8\"]\n+ cidr_blocks = [\"0.0.0.0/0\"]\n }\n",
66+
"expected_findings": [
67+
{
68+
"description": "Security group opens SSH to the entire internet.",
69+
"severity": "Error",
70+
"category": "Security",
71+
"file_pattern": "main.tf",
72+
"line_hint": 6,
73+
"contains_any": [
74+
"ssh open to the internet",
75+
"0.0.0.0/0",
76+
"world-open ingress",
77+
"publicly accessible ssh",
78+
"open to the world"
79+
],
80+
"rule_id": "sec.infra.terraform-public"
81+
}
82+
],
83+
"negative_findings": [
84+
{
85+
"description": "Avoid style-only comments.",
86+
"contains": "style"
87+
}
88+
],
89+
"min_total": 1,
90+
"max_total": 8,
91+
"description": "Infrastructure changes should not expose SSH to the public internet.",
92+
"source": "deep-review-suite"
93+
},
94+
{
95+
"name": "github-actions-event-script-injection",
96+
"category": "security",
97+
"language": "yaml",
98+
"difficulty": "Medium",
99+
"diff_content": "diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml\nindex 1111111..2222222 100644\n--- a/.github/workflows/triage.yml\n+++ b/.github/workflows/triage.yml\n@@ -1,6 +1,6 @@\n name: Issue Triage\n on: issues\n jobs:\n label:\n runs-on: ubuntu-latest\n- steps: []\n+ steps:\n+ - run: echo \"${{ github.event.issue.title }}\" | bash\n",
100+
"expected_findings": [
101+
{
102+
"description": "Attacker-controlled GitHub event data is executed by a shell step.",
103+
"severity": "Error",
104+
"category": "Security",
105+
"file_pattern": ".github/workflows/triage.yml",
106+
"line_hint": 7,
107+
"contains_any": [
108+
"script injection",
109+
"attacker-controlled issue title",
110+
"untrusted github event data",
111+
"piped into bash",
112+
"arbitrary command execution"
113+
],
114+
"rule_id": "sec.supply-chain.ci-injection"
115+
}
116+
],
117+
"negative_findings": [
118+
{
119+
"description": "Avoid style-only comments.",
120+
"contains": "style"
121+
}
122+
],
123+
"min_total": 1,
124+
"max_total": 8,
125+
"description": "CI automation should never pipe untrusted GitHub event fields into a shell.",
126+
"source": "deep-review-suite"
127+
},
128+
{
129+
"name": "k8s-privileged-container",
130+
"category": "security",
131+
"language": "yaml",
132+
"difficulty": "Easy",
133+
"diff_content": "diff --git a/deploy/pod.yaml b/deploy/pod.yaml\nindex 1111111..2222222 100644\n--- a/deploy/pod.yaml\n+++ b/deploy/pod.yaml\n@@ -5,6 +5,6 @@ spec:\n containers:\n - name: api\n image: example/api:latest\n securityContext:\n- privileged: false\n+ privileged: true\n ports:\n - containerPort: 8080\n",
134+
"expected_findings": [
135+
{
136+
"description": "Pod enables privileged container execution.",
137+
"severity": "Error",
138+
"category": "Security",
139+
"file_pattern": "deploy/pod.yaml",
140+
"line_hint": 6,
141+
"contains_any": [
142+
"privileged container",
143+
"securitycontext privileged",
144+
"full host access",
145+
"should not run privileged",
146+
"drop privileged"
147+
],
148+
"rule_id": "sec.infra.k8s-privileged"
149+
}
150+
],
151+
"negative_findings": [
152+
{
153+
"description": "Avoid style-only comments.",
154+
"contains": "style"
155+
}
156+
],
157+
"min_total": 1,
158+
"max_total": 8,
159+
"description": "Kubernetes workloads should avoid privileged mode unless absolutely required.",
160+
"source": "deep-review-suite"
161+
}
162+
]
163+
}

‎src/commands/eval/fixtures.rs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,4 +195,41 @@ expect:
195195
assert_eq!(fixtures[0].fixture.name.as_deref(), Some("community/panic"));
196196
assert_eq!(fixtures[1].fixture.name.as_deref(), Some("standard"));
197197
}
198+
199+
#[test]
200+
fn test_checked_in_infra_pack_loads_expected_fixtures() {
201+
let pack_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
202+
.join("eval/fixtures/deep_review_suite/review_depth_infra.json");
203+
204+
let fixtures = load_eval_fixtures_from_path(&pack_path).unwrap();
205+
206+
assert_eq!(fixtures.len(), 4);
207+
assert_eq!(
208+
fixtures[0].suite_name.as_deref(),
209+
Some("review-depth-infra")
210+
);
211+
assert_eq!(
212+
fixtures[0].fixture.name.as_deref(),
213+
Some("review-depth-infra/docker-user-root")
214+
);
215+
assert_eq!(
216+
fixtures[0].fixture.expect.must_find[0].rule_id.as_deref(),
217+
Some("sec.infra.docker-root")
218+
);
219+
assert_eq!(
220+
fixtures[1]
221+
.metadata
222+
.as_ref()
223+
.and_then(|metadata| metadata.language.as_deref()),
224+
Some("hcl")
225+
);
226+
assert_eq!(
227+
fixtures[2].fixture.expect.must_find[0].rule_id.as_deref(),
228+
Some("sec.supply-chain.ci-injection")
229+
);
230+
assert_eq!(
231+
fixtures[3].fixture.expect.must_find[0].rule_id.as_deref(),
232+
Some("sec.infra.k8s-privileged")
233+
);
234+
}
198235
}

0 commit comments

Comments
 (0)