|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +FAIL-reachable self-test for the WP-33 tri-state extension of wp18_conformance_gate.py. |
| 4 | +
|
| 5 | +Verifies that Check B now recognises THREE pack kinds (bitexact / bitexact_selfconsistent |
| 6 | +/ structural) and counts each separately, WITHOUT silently promoting a self-consistent |
| 7 | +pack to the stronger "bitexact" label. Every positive assertion is paired with a negative |
| 8 | +control (a planted defect that MUST flip the gate verdict), per the FAIL-reachable rule. |
| 9 | +
|
| 10 | +stdlib only, ZERO local imports beyond the gate under test. ASCII-only. Apache-2.0. |
| 11 | +
|
| 12 | +Run: python3 wp18_gate_selfconsistent_selftest.py |
| 13 | +Exit: 0 = all checks pass (incl. all negative controls flip), 1 = a check failed. |
| 14 | +""" |
| 15 | +import json |
| 16 | +import math |
| 17 | +import os |
| 18 | +import sys |
| 19 | +import tempfile |
| 20 | + |
| 21 | +HERE = os.path.dirname(os.path.abspath(__file__)) |
| 22 | +sys.path.insert(0, HERE) |
| 23 | +import wp18_conformance_gate as gate # noqa: E402 |
| 24 | + |
| 25 | +PASS = 0 |
| 26 | +FAIL = 0 |
| 27 | + |
| 28 | + |
| 29 | +def check(label, cond): |
| 30 | + global PASS, FAIL |
| 31 | + if cond: |
| 32 | + PASS += 1 |
| 33 | + print("PASS %s" % label) |
| 34 | + else: |
| 35 | + FAIL += 1 |
| 36 | + print("FAIL %s" % label) |
| 37 | + |
| 38 | + |
| 39 | +def _write(path, obj): |
| 40 | + with open(path, "w", encoding="ascii") as fh: |
| 41 | + json.dump(obj, fh) |
| 42 | + |
| 43 | + |
| 44 | +def _ssot(path, ids): |
| 45 | + with open(path, "w", encoding="ascii") as fh: |
| 46 | + for i in ids: |
| 47 | + fh.write("// CATALOG: id=%s\n" % i) |
| 48 | + |
| 49 | + |
| 50 | +def _bitexact_pack(name): |
| 51 | + # one finite row that re-derives exactly (abs_error 0), plus an inf special row |
| 52 | + return { |
| 53 | + "format": name, |
| 54 | + "vectors": [ |
| 55 | + {"name": "three", "input_f64": 3.0, "decoded_f64": 3.0, |
| 56 | + "abs_error": 0.0, "category": "normal"}, |
| 57 | + {"name": "pos_inf", "input_f64": "Infinity", "decoded_f64": "Infinity", |
| 58 | + "abs_error": 0.0, "category": "inf"}, |
| 59 | + ], |
| 60 | + } |
| 61 | + |
| 62 | + |
| 63 | +def _selfconsistent_pack(name): |
| 64 | + # dyadic-exact finite rows, abs_error 0 by construction, honest single-witness |
| 65 | + return { |
| 66 | + "format": name, |
| 67 | + "decode_provenance": "single decode law, no independent second witness", |
| 68 | + "vectors": [ |
| 69 | + {"name": "one", "input_f64": 1.0, "decoded_f64": 1.0, |
| 70 | + "abs_error": 0.0, "category": "normal"}, |
| 71 | + {"name": "two", "input_f64": 2.0, "decoded_f64": 2.0, |
| 72 | + "abs_error": 0.0, "category": "normal"}, |
| 73 | + ], |
| 74 | + } |
| 75 | + |
| 76 | + |
| 77 | +def build_tree(tmp, kinds): |
| 78 | + """kinds: list of (id, kind). Builds SSOT + vectors dir + INDEX with correct counts. |
| 79 | + Returns (ssot_path, vectors_dir, index_dict).""" |
| 80 | + vec = os.path.join(tmp, "vectors") |
| 81 | + os.makedirs(vec, exist_ok=True) |
| 82 | + ids = [k[0] for k in kinds] |
| 83 | + ssot = os.path.join(tmp, "formats_catalog.t27") |
| 84 | + _ssot(ssot, ids) |
| 85 | + packs = [] |
| 86 | + be = sc = st = 0 |
| 87 | + for pid, kind in kinds: |
| 88 | + fn = "%s_conformance_v0.json" % pid |
| 89 | + fpath = os.path.join(vec, fn) |
| 90 | + if kind == "structural": |
| 91 | + _write(fpath, {"format": pid, "vectors": []}) |
| 92 | + st += 1 |
| 93 | + elif kind == "bitexact_selfconsistent": |
| 94 | + _write(fpath, _selfconsistent_pack(pid)) |
| 95 | + sc += 1 |
| 96 | + else: |
| 97 | + _write(fpath, _bitexact_pack(pid)) |
| 98 | + be += 1 |
| 99 | + packs.append({"id": pid, "file": fn, "kind": kind, |
| 100 | + "n_vectors": 0 if kind == "structural" else 2, |
| 101 | + "sha256": gate._sha256_file(fpath)}) |
| 102 | + index = { |
| 103 | + "schema": "t27-conformance-index/v0.1", |
| 104 | + "total_formats": len(ids), |
| 105 | + "total_packs": len(packs), |
| 106 | + "bitexact_packs": be, |
| 107 | + "selfconsistent_packs": sc, |
| 108 | + "structural_packs": st, |
| 109 | + "packs": packs, |
| 110 | + } |
| 111 | + _write(os.path.join(vec, "INDEX_all_formats.json"), index) |
| 112 | + return ssot, vec, index |
| 113 | + |
| 114 | + |
| 115 | +def run(ssot, vec): |
| 116 | + code, report = gate.run_gate(ssot, vec, None) |
| 117 | + return code, report |
| 118 | + |
| 119 | + |
| 120 | +def b_check(report): |
| 121 | + return report["checks"]["B_index_counts"] |
| 122 | + |
| 123 | + |
| 124 | +def main(): |
| 125 | + # ----------------------------------------------------------------- T1 / T1b |
| 126 | + # T1: a tree with all three kinds + a correct INDEX -> CLEAN, recount matches. |
| 127 | + with tempfile.TemporaryDirectory() as tmp: |
| 128 | + ssot, vec, idx = build_tree(tmp, [ |
| 129 | + ("gf16", "bitexact"), |
| 130 | + ("gf48", "bitexact_selfconsistent"), |
| 131 | + ("gf96", "bitexact_selfconsistent"), |
| 132 | + ("decimal32", "structural"), |
| 133 | + ]) |
| 134 | + code, rep = run(ssot, vec) |
| 135 | + b = b_check(rep) |
| 136 | + check("T1 tri-state tree is CLEAN (exit 0)", code == 0) |
| 137 | + check("T1 recount [4,1,2,1] matches claimed", |
| 138 | + b["recount_total_bitexact_selfconsistent_structural"] == [4, 1, 2, 1] |
| 139 | + and b["ok"] is True) |
| 140 | + |
| 141 | + # T1b NEGATIVE CONTROL: claim the 2 selfconsistent packs as bitexact |
| 142 | + # (selfconsistent_packs=0, bitexact_packs=3) -> Check B MUST fail. |
| 143 | + idxp = json.load(open(os.path.join(vec, "INDEX_all_formats.json"), encoding="ascii")) |
| 144 | + idxp["bitexact_packs"] = 3 |
| 145 | + idxp["selfconsistent_packs"] = 0 |
| 146 | + _write(os.path.join(vec, "INDEX_all_formats.json"), idxp) |
| 147 | + code_b, rep_b = run(ssot, vec) |
| 148 | + check("T1b CONTROL mislabel sc-as-bitexact -> DRIFT (exit 2)", code_b == 2) |
| 149 | + check("T1b CONTROL Check B not ok", b_check(rep_b)["ok"] is False) |
| 150 | + |
| 151 | + # ----------------------------------------------------------------- T2 / T2b |
| 152 | + # T2: legacy 2-kind tree with NO selfconsistent key -> defaults to 0, CLEAN. |
| 153 | + with tempfile.TemporaryDirectory() as tmp: |
| 154 | + ssot, vec, idx = build_tree(tmp, [ |
| 155 | + ("gf16", "bitexact"), |
| 156 | + ("decimal32", "structural"), |
| 157 | + ]) |
| 158 | + # strip the optional key to simulate a legacy INDEX |
| 159 | + idxp = json.load(open(os.path.join(vec, "INDEX_all_formats.json"), encoding="ascii")) |
| 160 | + del idxp["selfconsistent_packs"] |
| 161 | + _write(os.path.join(vec, "INDEX_all_formats.json"), idxp) |
| 162 | + code, rep = run(ssot, vec) |
| 163 | + check("T2 legacy INDEX (no sc key) stays CLEAN (backward compat)", code == 0) |
| 164 | + check("T2 recount selfconsistent defaults to 0", |
| 165 | + b_check(rep)["recount_total_bitexact_selfconsistent_structural"] == [2, 1, 0, 1]) |
| 166 | + |
| 167 | + # T2b NEGATIVE CONTROL: same legacy tree but add a real selfconsistent pack on |
| 168 | + # disk + INDEX entry, leaving total/bitexact stale -> recount must catch it. |
| 169 | + vecdir = vec |
| 170 | + scfn = "gf48_conformance_v0.json" |
| 171 | + _write(os.path.join(vecdir, scfn), _selfconsistent_pack("gf48")) |
| 172 | + idxp = json.load(open(os.path.join(vecdir, "INDEX_all_formats.json"), encoding="ascii")) |
| 173 | + idxp["packs"].append({"id": "gf48", "file": scfn, "kind": "bitexact_selfconsistent", |
| 174 | + "n_vectors": 2, "sha256": gate._sha256_file(os.path.join(vecdir, scfn))}) |
| 175 | + # deliberately do NOT bump total_packs / selfconsistent_packs |
| 176 | + _write(os.path.join(vecdir, "INDEX_all_formats.json"), idxp) |
| 177 | + _ssot(ssot, ["gf16", "decimal32", "gf48"]) # keep Check A happy |
| 178 | + code_b, rep_b = run(ssot, vecdir) |
| 179 | + check("T2b CONTROL added sc pack w/ stale counts -> DRIFT (exit 2)", code_b == 2) |
| 180 | + |
| 181 | + # ----------------------------------------------------------------- T3 / T3b |
| 182 | + # T3: an UNKNOWN kind is drift, NOT silently folded into bitexact. |
| 183 | + with tempfile.TemporaryDirectory() as tmp: |
| 184 | + ssot, vec, idx = build_tree(tmp, [ |
| 185 | + ("gf16", "bitexact"), |
| 186 | + ("decimal32", "structural"), |
| 187 | + ]) |
| 188 | + idxp = json.load(open(os.path.join(vec, "INDEX_all_formats.json"), encoding="ascii")) |
| 189 | + idxp["packs"][0]["kind"] = "totally_made_up_kind" |
| 190 | + _write(os.path.join(vec, "INDEX_all_formats.json"), idxp) |
| 191 | + code, rep = run(ssot, vec) |
| 192 | + check("T3 unknown kind -> DRIFT (exit 2)", code == 2) |
| 193 | + check("T3 unknown_kind list is non-empty", |
| 194 | + len(b_check(rep)["unknown_kind"]) == 1) |
| 195 | + |
| 196 | + # T3b CONTROL: rename it back to the valid kind -> CLEAN again. |
| 197 | + idxp["packs"][0]["kind"] = "bitexact" |
| 198 | + _write(os.path.join(vec, "INDEX_all_formats.json"), idxp) |
| 199 | + code_b, rep_b = run(ssot, vec) |
| 200 | + check("T3b CONTROL valid kind restored -> CLEAN (exit 0)", code_b == 0) |
| 201 | + |
| 202 | + # ----------------------------------------------------------------- T4 / T4b |
| 203 | + # T4: Check D STILL runs on selfconsistent packs (they are not structural), so a |
| 204 | + # corrupted abs_error in a selfconsistent pack MUST be caught (no free pass). |
| 205 | + with tempfile.TemporaryDirectory() as tmp: |
| 206 | + ssot, vec, idx = build_tree(tmp, [ |
| 207 | + ("gf48", "bitexact_selfconsistent"), |
| 208 | + ]) |
| 209 | + code, rep = run(ssot, vec) |
| 210 | + check("T4 clean selfconsistent pack -> CLEAN (exit 0)", code == 0) |
| 211 | + |
| 212 | + # T4b CONTROL: lie about abs_error in a selfconsistent row -> Check D fires. |
| 213 | + scfile = os.path.join(vec, "gf48_conformance_v0.json") |
| 214 | + pk = json.load(open(scfile, encoding="ascii")) |
| 215 | + pk["vectors"][0]["abs_error"] = 0.5 # decoded==input==1.0 so true error is 0 |
| 216 | + _write(scfile, pk) |
| 217 | + # refresh sha so Check C does not pre-empt Check D |
| 218 | + idxp = json.load(open(os.path.join(vec, "INDEX_all_formats.json"), encoding="ascii")) |
| 219 | + idxp["packs"][0]["sha256"] = gate._sha256_file(scfile) |
| 220 | + _write(os.path.join(vec, "INDEX_all_formats.json"), idxp) |
| 221 | + code_b, rep_b = run(ssot, vec) |
| 222 | + check("T4b CONTROL corrupted abs_error in sc pack -> caught by D (exit 2)", code_b == 2) |
| 223 | + check("T4b CONTROL Check D not ok", |
| 224 | + rep_b["checks"]["D_rederive_abs_error"]["ok"] is False) |
| 225 | + |
| 226 | + print("") |
| 227 | + print("SELFTEST RESULT: %d PASS, %d FAIL" % (PASS, FAIL)) |
| 228 | + return 0 if FAIL == 0 else 1 |
| 229 | + |
| 230 | + |
| 231 | +if __name__ == "__main__": |
| 232 | + sys.exit(main()) |
0 commit comments