Skip to content

Commit 00c300e

Browse files
BryceWDesignBryceWDesign
authored
Implement integration tests for prototype capability rollup
Added integration tests for prototype capability rollup evaluation, ensuring readiness reports are generated correctly for various scenarios.
1 parent fb4b6fc commit 00c300e

1 file changed

Lines changed: 272 additions & 0 deletions

File tree

‎tests/test_prototype_rollup_12_capability_integration.py‎

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
from __future__ import annotations
2+
3+
from ix_autonomy_assurance_case_runtime.assurance_dossier_readiness import (
4+
AssuranceDossierLayerReadinessReport,
5+
AssuranceDossierReadinessDecision,
6+
)
7+
from ix_autonomy_assurance_case_runtime.assurance_dossier_validation import (
8+
AssuranceDossierValidationReport,
9+
)
10+
from ix_autonomy_assurance_case_runtime.claim_guardrails_readiness import (
11+
ClaimGuardrailLayerReadinessReport,
12+
ClaimGuardrailReadinessDecision,
13+
)
14+
from ix_autonomy_assurance_case_runtime.claim_guardrails_validation import (
15+
ClaimGuardrailValidationReport,
16+
)
17+
from ix_autonomy_assurance_case_runtime.export_package_readiness import (
18+
ExportPackageLayerReadinessReport,
19+
ExportPackageReadinessDecision,
20+
)
21+
from ix_autonomy_assurance_case_runtime.export_package_validation import (
22+
ExportPackageValidationReport,
23+
)
24+
from ix_autonomy_assurance_case_runtime.federal_evaluation_profile_readiness import (
25+
FederalEvaluationLayerReadinessReport,
26+
FederalEvaluationReadinessDecision,
27+
)
28+
from ix_autonomy_assurance_case_runtime.federal_evaluation_profile_validation import (
29+
FederalEvaluationValidationReport,
30+
)
31+
from ix_autonomy_assurance_case_runtime.framework_crosswalk import FrameworkCoverageReport
32+
from ix_autonomy_assurance_case_runtime.framework_crosswalk_evidence import (
33+
FrameworkEvidenceCoverageReport,
34+
)
35+
from ix_autonomy_assurance_case_runtime.framework_crosswalk_readiness import (
36+
FrameworkCrosswalkLayerReadinessReport,
37+
FrameworkCrosswalkReadinessDecision,
38+
)
39+
from ix_autonomy_assurance_case_runtime.monitoring_readiness import (
40+
MonitoringLayerReadinessReport,
41+
MonitoringReadinessDecision,
42+
)
43+
from ix_autonomy_assurance_case_runtime.monitoring_validation import MonitoringValidationReport
44+
from ix_autonomy_assurance_case_runtime.policy_readiness import (
45+
PolicyLayerReadinessDecision,
46+
PolicyLayerReadinessReport,
47+
)
48+
from ix_autonomy_assurance_case_runtime.policy_waiver_evidence import (
49+
PolicyWaiverEvidenceCoverageReport,
50+
)
51+
from ix_autonomy_assurance_case_runtime.prototype_readiness import (
52+
PrototypeClaimLevel,
53+
PrototypeReadinessDecision,
54+
)
55+
from ix_autonomy_assurance_case_runtime.prototype_rollup import (
56+
CapabilityLayerReport,
57+
PrototypeCapabilityRollupEvaluator,
58+
)
59+
from ix_autonomy_assurance_case_runtime.prototype_target import (
60+
required_serious_prototype_capability_ids,
61+
serious_prototype_capability_ids,
62+
)
63+
from ix_autonomy_assurance_case_runtime.provenance_readiness import (
64+
ProvenanceLayerReadinessReport,
65+
ProvenanceReadinessDecision,
66+
)
67+
from ix_autonomy_assurance_case_runtime.provenance_verifier import (
68+
ProvenanceManifestDecision,
69+
ProvenanceManifestVerificationReport,
70+
)
71+
from ix_autonomy_assurance_case_runtime.registry_catalog import RegistryValidationReport
72+
from ix_autonomy_assurance_case_runtime.registry_evidence import RegistryEvidenceCoverageReport
73+
from ix_autonomy_assurance_case_runtime.registry_readiness import (
74+
RegistryLayerReadinessReport,
75+
RegistryReadinessDecision,
76+
)
77+
from ix_autonomy_assurance_case_runtime.review_workflow_readiness import (
78+
ReviewWorkflowLayerReadinessReport,
79+
ReviewWorkflowReadinessDecision,
80+
)
81+
from ix_autonomy_assurance_case_runtime.review_workflow_validation import (
82+
ReviewWorkflowValidationReport,
83+
)
84+
from ix_autonomy_assurance_case_runtime.scenario_campaign_readiness import (
85+
ScenarioCampaignLayerReadinessReport,
86+
ScenarioCampaignReadinessDecision,
87+
)
88+
from ix_autonomy_assurance_case_runtime.telemetry_readiness import (
89+
TelemetryLayerReadinessReport,
90+
TelemetryReadinessDecision,
91+
)
92+
93+
94+
def _complete_layer_reports() -> tuple[CapabilityLayerReport, ...]:
95+
return (
96+
RegistryLayerReadinessReport(
97+
decision=RegistryReadinessDecision.COMPLETE,
98+
catalog_report=RegistryValidationReport(
99+
model_count=1,
100+
system_count=1,
101+
use_case_count=1,
102+
deployment_count=1,
103+
findings=(),
104+
),
105+
evidence_report=RegistryEvidenceCoverageReport(
106+
referenced_bundle_count=3,
107+
provided_bundle_count=3,
108+
findings=(),
109+
),
110+
findings=(),
111+
),
112+
PolicyLayerReadinessReport(
113+
decision=PolicyLayerReadinessDecision.COMPLETE,
114+
evaluation_reports=(),
115+
waiver_evidence_report=PolicyWaiverEvidenceCoverageReport(
116+
policy_pack_id="pack-fed-001",
117+
waiver_count=1,
118+
referenced_bundle_count=1,
119+
provided_bundle_count=1,
120+
findings=(),
121+
),
122+
findings=(),
123+
),
124+
FrameworkCrosswalkLayerReadinessReport(
125+
decision=FrameworkCrosswalkReadinessDecision.COMPLETE,
126+
coverage_report=FrameworkCoverageReport(coverage=(), findings=()),
127+
evidence_report=FrameworkEvidenceCoverageReport(
128+
referenced_bundle_count=1,
129+
provided_bundle_count=1,
130+
findings=(),
131+
),
132+
findings=(),
133+
),
134+
ProvenanceLayerReadinessReport(
135+
decision=ProvenanceReadinessDecision.COMPLETE,
136+
verification_report=ProvenanceManifestVerificationReport(
137+
manifest_id="manifest-provenance-001",
138+
decision=ProvenanceManifestDecision.VERIFIED,
139+
artifact_count=1,
140+
signed_artifact_count=1,
141+
attestation_count=1,
142+
findings=(),
143+
),
144+
findings=(),
145+
),
146+
TelemetryLayerReadinessReport(
147+
decision=TelemetryReadinessDecision.COMPLETE,
148+
source_count=1,
149+
schema_count=1,
150+
replay_record_count=1,
151+
adapter_report_count=1,
152+
findings=(),
153+
),
154+
ScenarioCampaignLayerReadinessReport(
155+
decision=ScenarioCampaignReadinessDecision.COMPLETE,
156+
campaign_count=1,
157+
run_report_count=1,
158+
findings=(),
159+
),
160+
MonitoringLayerReadinessReport(
161+
decision=MonitoringReadinessDecision.COMPLETE,
162+
validation_report=MonitoringValidationReport(
163+
snapshot_count=1,
164+
drift_count=1,
165+
incident_count=1,
166+
trigger_count=1,
167+
findings=(),
168+
),
169+
findings=(),
170+
),
171+
ReviewWorkflowLayerReadinessReport(
172+
decision=ReviewWorkflowReadinessDecision.COMPLETE,
173+
validation_report=ReviewWorkflowValidationReport(
174+
workflow_id="workflow-review-001",
175+
finding_count=1,
176+
signoff_count=1,
177+
dissent_count=0,
178+
evidence_bundle_count=1,
179+
findings=(),
180+
),
181+
findings=(),
182+
),
183+
ExportPackageLayerReadinessReport(
184+
decision=ExportPackageReadinessDecision.COMPLETE,
185+
validation_report=ExportPackageValidationReport(
186+
package_id="export-package-001",
187+
artifact_count=5,
188+
redaction_rule_count=1,
189+
evidence_bundle_count=3,
190+
provenance_manifest_count=1,
191+
findings=(),
192+
),
193+
findings=(),
194+
),
195+
AssuranceDossierLayerReadinessReport(
196+
decision=AssuranceDossierReadinessDecision.COMPLETE,
197+
validation_report=AssuranceDossierValidationReport(
198+
dossier_id="dossier-runtime-001",
199+
trace_thread_count=1,
200+
artifact_count=5,
201+
evidence_reference_count=4,
202+
evidence_bundle_count=4,
203+
provenance_manifest_count=1,
204+
export_package_count=1,
205+
findings=(),
206+
),
207+
findings=(),
208+
),
209+
ClaimGuardrailLayerReadinessReport(
210+
decision=ClaimGuardrailReadinessDecision.COMPLETE,
211+
validation_report=ClaimGuardrailValidationReport(
212+
package_id="claim-package-runtime-001",
213+
claim_count=2,
214+
evidence_reference_count=1,
215+
prohibited_rule_count=1,
216+
evidence_bundle_count=1,
217+
findings=(),
218+
),
219+
findings=(),
220+
),
221+
FederalEvaluationLayerReadinessReport(
222+
decision=FederalEvaluationReadinessDecision.COMPLETE,
223+
validation_report=FederalEvaluationValidationReport(
224+
profile_id="fed-profile-runtime-001",
225+
mapping_count=12,
226+
concern_count=8,
227+
core_concern_count=8,
228+
completed_capability_count=12,
229+
artifact_count=8,
230+
evidence_bundle_count=8,
231+
findings=(),
232+
),
233+
findings=(),
234+
),
235+
)
236+
237+
238+
def test_all_twelve_capability_readiness_surfaces_roll_up_to_full_local_maturity() -> None:
239+
report = PrototypeCapabilityRollupEvaluator().evaluate(
240+
_complete_layer_reports(),
241+
requested_claim_level=PrototypeClaimLevel.SERIOUS_OPEN_SOURCE_PROTOTYPE,
242+
)
243+
244+
assert report.is_rollup_clean()
245+
assert report.readiness_report.decision is PrototypeReadinessDecision.ALLOW
246+
assert report.achieved_percent == 100
247+
assert report.completed_capability_ids() == serious_prototype_capability_ids()
248+
assert report.missing_expected_capability_ids == ()
249+
assert report.unexpected_completed_capability_ids == ()
250+
assert report.duplicate_capability_ids == ()
251+
assert report.findings == ()
252+
assert report.summary() == (
253+
"prototype-rollup: 100% (12 completed capability(s), 0 blocker(s), "
254+
"0 warning(s), target=80%)"
255+
)
256+
257+
258+
def test_original_nine_required_readiness_surfaces_still_reach_eighty_percent() -> None:
259+
report = PrototypeCapabilityRollupEvaluator().evaluate(
260+
_complete_layer_reports()[:9],
261+
requested_claim_level=PrototypeClaimLevel.SERIOUS_OPEN_SOURCE_PROTOTYPE,
262+
expected_capability_ids=required_serious_prototype_capability_ids(),
263+
)
264+
265+
assert report.is_rollup_clean()
266+
assert report.readiness_report.decision is PrototypeReadinessDecision.ALLOW
267+
assert report.achieved_percent == 80
268+
assert report.completed_capability_ids() == required_serious_prototype_capability_ids()
269+
assert report.missing_expected_capability_ids == ()
270+
assert report.unexpected_completed_capability_ids == ()
271+
assert report.duplicate_capability_ids == ()
272+
assert report.findings == ()

0 commit comments

Comments
 (0)