|
8 | 8 | workflow_dispatch: |
9 | 9 | inputs: |
10 | 10 | test_filter: |
11 | | - description: 'Filter by category (all, baseline, create_risk, lower_risk, discover, instruct)' |
| 11 | + description: 'Filter by category (all, create_risk, lower_risk, discover, instruct)' |
12 | 12 | required: false |
13 | 13 | type: choice |
14 | 14 | default: 'all' |
15 | 15 | options: |
16 | 16 | - 'all' |
17 | | - - 'baseline' |
18 | 17 | - 'create_risk' |
19 | 18 | - 'lower_risk' |
20 | 19 | - 'discover' |
|
51 | 50 | # effect for this scenario? |
52 | 51 | # |
53 | 52 | # Categories: |
54 | | -# baseline - Same scenarios as scale-test nightly, but WITH knowledge |
55 | | -# present. Compared to scale-test nightly (no knowledge) to |
56 | | -# measure overall knowledge impact. |
57 | 53 | # create_risk - Knowledge should cause new or elevated risks |
58 | 54 | # lower_risk - Knowledge should reduce or disprove risks |
59 | 55 | # discover - Knowledge should surface resources not normally found |
60 | 56 | # instruct - Knowledge should add operational context to risks |
61 | 57 | # |
62 | | -# Note: baseline and category tests use the SAME knowledge files (all 5 are |
63 | | -# always present). The categories describe what we EXPECT the dominant effect |
64 | | -# to be for each scenario based on which knowledge files are most relevant. |
| 58 | +# Baselines (no-knowledge runs) are provided by the nightly scale-test |
| 59 | +# workflow, which runs the same scenarios without knowledge files. The scale |
| 60 | +# dashboard compares knowledge test results against those nightly runs. |
| 61 | +# |
| 62 | +# All knowledge files (all 5) are always present. The categories describe |
| 63 | +# what we EXPECT the dominant effect to be for each scenario based on which |
| 64 | +# knowledge files are most relevant. |
65 | 65 | # ============================================================================= |
66 | 66 |
|
67 | 67 | jobs: |
@@ -164,42 +164,6 @@ jobs: |
164 | 164 |
|
165 | 165 | matrix: |
166 | 166 | include: |
167 | | - # ----------------------------------------------------------------- |
168 | | - # Baselines WITH knowledge |
169 | | - # Same scenarios as nightly scale-test, but now with knowledge files |
170 | | - # present. Compare these results to the nightly (no-knowledge) runs |
171 | | - # to see the overall impact of having knowledge available. |
172 | | - # ----------------------------------------------------------------- |
173 | | - - test_id: with-knowledge-sg-open |
174 | | - scenario: shared_sg_open |
175 | | - category: baseline |
176 | | - expected_effect: "Baseline with knowledge present. Compare to nightly no-knowledge run for same scenario." |
177 | | - relevant_knowledge: "security-standards.md (network access rules, shared SG context), change-approvals.md (firewall exception process)" |
178 | | - |
179 | | - - test_id: with-knowledge-lambda-timeout |
180 | | - scenario: lambda_timeout |
181 | | - category: baseline |
182 | | - expected_effect: "Baseline with knowledge present. Interesting because two files have contradictory guidance (platform says 180s minimum, infra-guide says dummy functions dont need it)." |
183 | | - relevant_knowledge: "platform-event-pipeline.md (SQS timeout rules), infrastructure-guide.md (scale test dummy functions)" |
184 | | - |
185 | | - - test_id: with-knowledge-vpc-peering |
186 | | - scenario: vpc_peering_change |
187 | | - category: baseline |
188 | | - expected_effect: "Baseline with knowledge present. Compare to nightly no-knowledge run for same scenario." |
189 | | - relevant_knowledge: "multi-region-design.md (VPC peering DNS is approved), change-approvals.md (cross-region sign-offs)" |
190 | | - |
191 | | - - test_id: with-knowledge-sns-change |
192 | | - scenario: central_sns_change |
193 | | - category: baseline |
194 | | - expected_effect: "Baseline with knowledge present. Compare to nightly no-knowledge run for same scenario." |
195 | | - relevant_knowledge: "platform-event-pipeline.md (SNS hardening approved, Lambda publishers), multi-region-design.md (central resources, SSM references)" |
196 | | - |
197 | | - - test_id: with-knowledge-kms-orphan |
198 | | - scenario: kms_orphan_simulation |
199 | | - category: baseline |
200 | | - expected_effect: "Baseline with knowledge present. Compare to nightly no-knowledge run for same scenario." |
201 | | - relevant_knowledge: "security-standards.md (KMS process, encryption requirements), change-approvals.md (SEC-REVIEW tickets)" |
202 | | - |
203 | 167 | # ----------------------------------------------------------------- |
204 | 168 | # Create risk |
205 | 169 | # For these scenarios, specific knowledge files should cause Overmind |
|
0 commit comments