Skip to content

Commit ce7b202

Browse files
BryceWDesignBryceWDesign
authored
Add regression tests for IX Cognition CLI
1 parent 9946b9a commit ce7b202

1 file changed

Lines changed: 261 additions & 0 deletions

File tree

‎tests/test_ix_cognition_cli_regression.py‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
import io
2+
import json
3+
import tempfile
4+
import unittest
5+
from contextlib import redirect_stderr, redirect_stdout
6+
from pathlib import Path
7+
8+
from ix.cli import main
9+
10+
11+
REPO_ROOT = Path(__file__).resolve().parents[1]
12+
EXAMPLES_DIR = REPO_ROOT / "examples"
13+
CANONICAL_EXAMPLE = EXAMPLES_DIR / "cognitionkernel_wave6_contract.ix"
14+
15+
NEGATIVE_EXAMPLE_EXPECTATIONS = {
16+
"cognitionkernel_wave6_missing_obligations.ix": (
17+
"cognition_contract.required_obligations.missing",
18+
),
19+
"cognitionkernel_wave6_overclaiming.ix": (
20+
"cognition_contract.prohibited_claim_language.present",
21+
"cognition_contract.required_obligations.missing",
22+
),
23+
"cognitionkernel_wave6_wrong_handoff.ix": (
24+
"cognition_contract.kernel_handoff.missing",
25+
"cognition_contract.required_obligations.missing",
26+
),
27+
"cognitionkernel_wave6_noncanonical_falsification.ix": (
28+
"cognition_contract.obligation_canonical_falsification.missing",
29+
"cognition_contract.required_obligations.missing",
30+
),
31+
}
32+
33+
COGNITION_ARTIFACTS = {
34+
"contract.json",
35+
"obligations.json",
36+
"falsification-gates.json",
37+
"claim-boundaries.json",
38+
"kernel-handoff.json",
39+
"satisfaction-report.json",
40+
"failure-report.json",
41+
}
42+
43+
44+
class TestIXCognitionCliRegression(unittest.TestCase):
45+
def test_cli_about_and_version_remain_available(self):
46+
about_stdout = io.StringIO()
47+
with redirect_stdout(about_stdout):
48+
about_code = main(["about"])
49+
50+
self.assertEqual(about_code, 0)
51+
self.assertIn("IX", about_stdout.getvalue())
52+
self.assertIn("audit-first", about_stdout.getvalue())
53+
54+
version_stdout = io.StringIO()
55+
with redirect_stdout(version_stdout):
56+
version_code = main(["version"])
57+
58+
self.assertEqual(version_code, 0)
59+
self.assertTrue(version_stdout.getvalue().strip())
60+
61+
def test_cli_run_trace_and_test_keep_cognition_contract_metadata_only(self):
62+
run_stdout = io.StringIO()
63+
with redirect_stdout(run_stdout):
64+
run_code = main(["run", str(CANONICAL_EXAMPLE)])
65+
66+
self.assertEqual(run_code, 0)
67+
self.assertEqual(run_stdout.getvalue(), "")
68+
69+
trace_stdout = io.StringIO()
70+
with redirect_stdout(trace_stdout):
71+
trace_code = main(["trace", str(CANONICAL_EXAMPLE)])
72+
73+
self.assertEqual(trace_code, 0)
74+
trace_payload = json.loads(trace_stdout.getvalue())
75+
self.assertEqual(trace_payload["outputs"], [])
76+
self.assertEqual(trace_payload["replies"], [])
77+
self.assertEqual(trace_payload["approvals_required"], [])
78+
self.assertEqual(trace_payload["contract_metadata"]["counts"]["attempts"], 1)
79+
self.assertEqual(trace_payload["contract_metadata"]["counts"]["obligations"], 20)
80+
self.assertEqual(
81+
trace_payload["contract_metadata"]["runtime_semantics"],
82+
"metadata_only_not_executed",
83+
)
84+
self.assertEqual(
85+
[event["kind"] for event in trace_payload["trace"]],
86+
["run.start", "contract.metadata", "run.complete"],
87+
)
88+
89+
test_stdout = io.StringIO()
90+
with redirect_stdout(test_stdout):
91+
test_code = main(["test", str(CANONICAL_EXAMPLE)])
92+
93+
self.assertEqual(test_code, 0)
94+
self.assertIn("PASS: 0 assertion(s), 3 trace event(s)", test_stdout.getvalue())
95+
96+
def test_cli_assure_text_mode_passes_canonical_cognition_contract(self):
97+
stdout = io.StringIO()
98+
with redirect_stdout(stdout):
99+
code = main(
100+
[
101+
"assure",
102+
str(CANONICAL_EXAMPLE),
103+
"--profile",
104+
"cognitionkernel-wave6",
105+
]
106+
)
107+
108+
output = stdout.getvalue()
109+
self.assertEqual(code, 0)
110+
self.assertIn("ASSURANCE PASS", output)
111+
self.assertIn("PROFILE: cognitionkernel-wave6", output)
112+
self.assertIn("cognition_contract.required_obligations.present", output)
113+
self.assertIn("cognition_contract.obligations.canonical", output)
114+
self.assertIn("cognition_contract.agi_claim_restriction.present", output)
115+
self.assertIn("cognition_contract.research_boundary.present", output)
116+
self.assertIn("cognition_contract.prohibited_claim_language.absent", output)
117+
self.assertIn(
118+
"cognition_contract.obligation_canonical_falsification.present",
119+
output,
120+
)
121+
122+
def test_cli_assure_text_mode_fails_negative_cognition_examples(self):
123+
for example_name, expected_checks in NEGATIVE_EXAMPLE_EXPECTATIONS.items():
124+
with self.subTest(example=example_name):
125+
stdout = io.StringIO()
126+
example_file = EXAMPLES_DIR / example_name
127+
128+
with redirect_stdout(stdout):
129+
code = main(
130+
[
131+
"assure",
132+
str(example_file),
133+
"--profile",
134+
"cognitionkernel-wave6",
135+
]
136+
)
137+
138+
output = stdout.getvalue()
139+
self.assertEqual(code, 2)
140+
self.assertIn("ASSURANCE FAIL", output)
141+
self.assertIn("PROFILE: cognitionkernel-wave6", output)
142+
for expected_check in expected_checks:
143+
self.assertIn(expected_check, output)
144+
145+
def test_cli_assure_json_mode_reports_profile_specific_separation(self):
146+
cognition_stdout = io.StringIO()
147+
with redirect_stdout(cognition_stdout):
148+
cognition_code = main(
149+
[
150+
"assure",
151+
str(CANONICAL_EXAMPLE),
152+
"--profile",
153+
"cognitionkernel-wave6",
154+
"--json",
155+
]
156+
)
157+
158+
self.assertEqual(cognition_code, 0)
159+
cognition_payload = json.loads(cognition_stdout.getvalue())
160+
cognition_check_ids = {check["id"] for check in cognition_payload["checks"]}
161+
self.assertEqual(cognition_payload["status"], "pass")
162+
self.assertIn(
163+
"cognition_contract.required_obligations.present",
164+
cognition_check_ids,
165+
)
166+
167+
local_stdout = io.StringIO()
168+
with redirect_stdout(local_stdout):
169+
local_code = main(
170+
[
171+
"assure",
172+
str(CANONICAL_EXAMPLE),
173+
"--profile",
174+
"experimental-local",
175+
"--json",
176+
]
177+
)
178+
179+
self.assertEqual(local_code, 2)
180+
local_payload = json.loads(local_stdout.getvalue())
181+
local_check_ids = {check["id"] for check in local_payload["checks"]}
182+
self.assertEqual(local_payload["profile"], "experimental-local")
183+
self.assertEqual(local_payload["status"], "fail")
184+
self.assertIn("program.no_executable_path", local_check_ids)
185+
self.assertNotIn(
186+
"cognition_contract.required_obligations.present",
187+
local_check_ids,
188+
)
189+
self.assertNotIn(
190+
"cognition_contract.required_obligations.missing",
191+
local_check_ids,
192+
)
193+
194+
def test_cli_evidence_exports_manifested_cognition_artifacts(self):
195+
with tempfile.TemporaryDirectory() as tmpdir:
196+
output_dir = Path(tmpdir) / "canonical-cognition-evidence"
197+
stdout = io.StringIO()
198+
199+
with redirect_stdout(stdout):
200+
code = main(
201+
[
202+
"evidence",
203+
str(CANONICAL_EXAMPLE),
204+
"--out",
205+
str(output_dir),
206+
]
207+
)
208+
209+
self.assertEqual(code, 0)
210+
output = stdout.getvalue()
211+
self.assertIn("EVIDENCE BUNDLE WRITTEN", output)
212+
213+
for artifact_name in COGNITION_ARTIFACTS:
214+
with self.subTest(artifact=artifact_name):
215+
self.assertIn(f"- {artifact_name}", output)
216+
self.assertTrue((output_dir / artifact_name).exists())
217+
218+
manifest = json.loads((output_dir / "manifest.json").read_text(encoding="utf-8"))
219+
summary = json.loads((output_dir / "summary.json").read_text(encoding="utf-8"))
220+
kernel_handoff = json.loads(
221+
(output_dir / "kernel-handoff.json").read_text(encoding="utf-8")
222+
)
223+
satisfaction = json.loads(
224+
(output_dir / "satisfaction-report.json").read_text(encoding="utf-8")
225+
)
226+
failure = json.loads(
227+
(output_dir / "failure-report.json").read_text(encoding="utf-8")
228+
)
229+
230+
self.assertTrue(COGNITION_ARTIFACTS.issubset(set(manifest["artifact_files"])))
231+
self.assertEqual(summary["counts"]["contract_attempts"], 1)
232+
self.assertEqual(summary["counts"]["contract_obligations"], 20)
233+
self.assertEqual(summary["counts"]["kernel_handoff_packages"], 1)
234+
self.assertEqual(summary["counts"]["satisfaction_report_items"], 20)
235+
self.assertEqual(summary["counts"]["failure_report_gates"], 20)
236+
self.assertEqual(kernel_handoff["packages"][0]["target"], "IX-CognitionKernel")
237+
self.assertEqual(satisfaction["status"], "not_evaluated")
238+
self.assertEqual(failure["status"], "not_evaluated")
239+
240+
def test_cli_unknown_cognition_profile_still_fails_closed_on_stderr(self):
241+
stdout = io.StringIO()
242+
stderr = io.StringIO()
243+
244+
with redirect_stdout(stdout), redirect_stderr(stderr):
245+
code = main(
246+
[
247+
"assure",
248+
str(CANONICAL_EXAMPLE),
249+
"--profile",
250+
"not-a-profile",
251+
]
252+
)
253+
254+
self.assertEqual(code, 2)
255+
self.assertEqual(stdout.getvalue(), "")
256+
self.assertIn("IX assure failed", stderr.getvalue())
257+
self.assertIn("Unknown assurance profile", stderr.getvalue())
258+
259+
260+
if __name__ == "__main__":
261+
unittest.main()

0 commit comments

Comments
 (0)