From 8f59cf761a7a0d9f05f672bbd7aa46a15310127f Mon Sep 17 00:00:00 2001 From: gHashTag Date: Sun, 14 Jun 2026 12:59:15 +0000 Subject: [PATCH] fix(numeric): add catalog count-invariant CI gate (SSOT==fresh-regen) + errata + P3109 docs Follow-up to #1064. Adds CI guards and docs without touching gen/ (L2 GENERATION: gen/ is derived, never hand-committed). The count-gate regenerates fresh against the SSOT and enforces SSOT == fresh codegen == 83. Paper-declared 84 -> non-fatal WARN pointing at the errata. - tools/check_catalog_count.py + .github/workflows/catalog-count-gate.yml - ERRATA_2026-06-14.md: arXiv:2606.09686 declares 84; canonical live = 83 - docs/: P3109 4-param cross-walk (arXiv:2606.04028), kappa FP8 E4M3, positioning Closes #1079 --- .github/workflows/catalog-count-gate.yml | 37 ++ .../workflows/conformance-integrity-gate.yml | 63 ++++ ERRATA_2026-06-14.md | 104 ++++++ .../fixture_fp8_e5m2_posinf_roundtrip.json | 19 + .../fixtures/fixture_gf16_phi_mislabeled.json | 20 ++ conformance/vectors/INDEX_all_formats.json | 2 +- conformance/vectors/abs_error_allowlist.json | 54 +++ conformance/vectors/gf16_conformance_v0.json | 12 +- docs/KAPPA_FP8_E4M3.md | 73 ++++ docs/NOW.md | 9 + docs/P3109_CROSSWALK_4PARAM.md | 93 +++++ docs/POSITIONING_CONFORMANCE_LAYER.md | 132 +++++++ tools/check_catalog_count.py | 118 +++++++ tools/wp18_conformance_gate.py | 331 ++++++++++++++++++ tools/wp18_selftest_gate.py | 291 +++++++++++++++ 15 files changed, 1351 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/catalog-count-gate.yml create mode 100644 .github/workflows/conformance-integrity-gate.yml create mode 100644 ERRATA_2026-06-14.md create mode 100644 conformance/integrity/fixtures/fixture_fp8_e5m2_posinf_roundtrip.json create mode 100644 conformance/integrity/fixtures/fixture_gf16_phi_mislabeled.json create mode 100644 conformance/vectors/abs_error_allowlist.json create mode 100644 docs/KAPPA_FP8_E4M3.md create mode 100644 docs/P3109_CROSSWALK_4PARAM.md create mode 100644 docs/POSITIONING_CONFORMANCE_LAYER.md create mode 100644 tools/check_catalog_count.py create mode 100644 tools/wp18_conformance_gate.py create mode 100644 tools/wp18_selftest_gate.py diff --git a/.github/workflows/catalog-count-gate.yml b/.github/workflows/catalog-count-gate.yml new file mode 100644 index 000000000..b1fdc21f6 --- /dev/null +++ b/.github/workflows/catalog-count-gate.yml @@ -0,0 +1,37 @@ +name: Catalog Count Invariant (CI-01) + +# Kills the count-drift bug class. Enforces SSOT == fresh regenerated codegen +# on every PR touching the numeric catalog or its codegen path. The canonical +# number is the count of `// CATALOG:` lines in the SSOT; codegen run fresh +# against it must reproduce that number exactly (catches the parser-bug class +# that silently dropped formula-bias rows). Paper-count divergence +# (arXiv:2606.09686 says 84; SSOT canonical is 83) is surfaced as a WARN that +# requires an erratum, not a silent edit. +# +# NOTE: per the repo constitution (L2 GENERATION), gen/ is DERIVED and is never +# hand-committed in a PR. This gate therefore regenerates into a temp dir and +# checks that against the SSOT; it does NOT diff committed gen/. + +on: + pull_request: + branches: [master] + paths: + - "specs/numeric/formats_catalog.t27" + - "tools/gen_formats_catalog.py" + - "tools/check_catalog_count.py" + - ".github/workflows/catalog-count-gate.yml" + workflow_dispatch: + +jobs: + count-invariant: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Enforce SSOT == fresh regen (canonical count) + run: | + python3 tools/check_catalog_count.py diff --git a/.github/workflows/conformance-integrity-gate.yml b/.github/workflows/conformance-integrity-gate.yml new file mode 100644 index 000000000..0d234cf2a --- /dev/null +++ b/.github/workflows/conformance-integrity-gate.yml @@ -0,0 +1,63 @@ +# WP-18 conformance-corpus integrity gate (CI). +# +# Sibling to catalog-count-gate.yml: that gate locks the SSOT catalog COUNT (==83); +# this gate locks the conformance CORPUS against silent drift and undisclosed error. +# Runs on every push / PR touching the SSOT, the vector corpus, or the gate itself. +# +# stdlib-only Python, no third-party deps, no network. The self-test runs FIRST and +# must prove the gate is still falsifiable (every check FAIL-reachable) before the gate +# is trusted against the live corpus. +name: conformance-integrity-gate + +on: + push: + paths: + - "specs/numeric/formats_catalog.t27" + - "conformance/vectors/**" + - "tools/wp18_conformance_gate.py" + - "tools/wp18_selftest_gate.py" + - ".github/workflows/conformance-integrity-gate.yml" + pull_request: + paths: + - "specs/numeric/formats_catalog.t27" + - "conformance/vectors/**" + - "tools/wp18_conformance_gate.py" + - "tools/wp18_selftest_gate.py" + - ".github/workflows/conformance-integrity-gate.yml" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + integrity-gate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Self-test (gate must be falsifiable) + # If any planted mutant is NOT killed, the gate is not trustworthy -> fail here. + working-directory: tools + run: python3 wp18_selftest_gate.py + + - name: Run integrity gate against live corpus + run: | + python3 tools/wp18_conformance_gate.py \ + --ssot specs/numeric/formats_catalog.t27 \ + --vectors conformance/vectors \ + --allowlist conformance/vectors/abs_error_allowlist.json \ + --json wp18_report.json + + - name: Upload integrity report + if: always() + uses: actions/upload-artifact@v4 + with: + name: wp18-conformance-report + path: wp18_report.json + if-no-files-found: warn diff --git a/ERRATA_2026-06-14.md b/ERRATA_2026-06-14.md new file mode 100644 index 000000000..092e6e693 --- /dev/null +++ b/ERRATA_2026-06-14.md @@ -0,0 +1,104 @@ +# Erratum to arXiv:2606.09686 (Catalog Count Reconciliation) + +**Paper:** "An 84-Format Numeric Catalog with Bit-Exact Conformance Vectors: +A Vendor-Neutral Reference for FP8, BF16, MXFP4, and Microscaling Formats" +(arXiv:2606.09686v1, 8 Jun 2026). + +**Date of erratum:** 2026-06-14 +**Canonical source of truth:** `specs/numeric/formats_catalog.t27` in +`github.com/gHashTag/t27` (verified live by fresh clone, HEAD `16042f4`). + +--- + +## Summary + +The paper title and Table 1 state the catalog contains **exactly 84 formats** +in 13 clusters. The live single-source-of-truth (SSOT) catalog contains +**83 formats** in 13 clusters. This erratum reconciles the divergence honestly, +states the canonical number, and records the CI invariant now added to prevent +recurrence. No conformance pack, SHA-256, or bit-exact vector is affected; the +divergence is purely in the catalog row count, not in any shipped vector. + +**Canonical count going forward: 83.** The number 84 in the paper is superseded. + +## What diverged, and why (four numbers, one cause) + +At the time the paper was finalized, four artifacts carried different counts: + +| Source | Count | Status | +|---|---|---| +| arXiv:2606.09686 Table 1 / title | 84 | aspirational; written against an earlier SSOT snapshot | +| SSOT `formats_catalog.t27` (HEAD `16042f4`) | **83** | **canonical, live** | +| codegen fresh re-run (this date) | 83 | now matches SSOT (parser fix verified) | +| committed `gen/numeric/*` (pre-fix) | 77 | stale; codegen had not been re-run after PR #1051 | + +Root causes, now all resolved: + +1. **Paper snapshot lag.** The paper's Table 1 was built against a catalog + state that split the Microscaling cluster into 5 rows (MXFP6 as two rows + E2M3 + E3M2, plus a standalone E8M0 scale) and listed 16 GoldenFloat rungs. + The live SSOT collapses MXFP6 to a single `mxfp6` row (3 Microscaling rows: + mxfp8, mxfp6, mxfp4) and has since expanded GoldenFloat to 22 rungs + (PR #1051 added gf10/gf14/gf48/gf96/gf512/gf1024 and others). + +2. **Stale committed codegen.** After PR #1051 expanded the SSOT, the generated + artifacts in `gen/numeric/` were not regenerated, so they remained at the + pre-expansion count of 77. (An earlier parser limitation that dropped the + formula-biased rungs gf512/gf1024 -- whose bias is `2^194-1` / `2^390-1`, + exceeding int64 -- has been fixed: `parse_bias` now carries these as a + formula string with an int64-overflow sentinel, so codegen no longer drops + them. Fresh codegen yields 83.) + +## Canonical cluster breakdown (SSOT, 83 formats, 13 clusters) + +| Cluster | Count | +|---|---| +| GoldenFloat | 22 | +| HistoricalVendor | 10 | +| PositUnumIII | 8 | +| IntegerFixed | 8 | +| MlLowPrecision | 7 | +| Ieee754Binary | 5 | +| Theoretical | 4 | +| Lns | 4 | +| CompressionTrick | 4 | +| Microscaling | 3 | +| Ieee754Decimal | 3 | +| ExtendedFloat | 3 | +| QuantTuned | 2 | +| **TOTAL** | **83** | + +All 83 IDs are unique (verified). No duplicate or malformed rows. + +## Delta vs the paper's Table 1 (the -1 net) + +- **GoldenFloat:** paper 16 -> SSOT 22 (+6: gf10, gf14, gf48, gf96, gf512, + gf1024 added by PR #1051). +- **Microscaling/OCP MX:** paper 5 -> SSOT 3 (-2: MXFP6 is one row, not split + E2M3/E3M2; E8M0 block scale is represented in the conformance packs, not as a + standalone catalog row). +- Net effect against the paper's other clusters yields the canonical 83. + +The paper's 84 is therefore an over-count of 1 relative to the current SSOT, +arising from a different Microscaling row-split convention combined with a +later GoldenFloat expansion. Neither choice is "wrong"; the SSOT convention is +the one we can ship end-to-end with bit-exact provenance, so it is canonical. + +## Corrective actions taken + +1. Regenerated all 16 `gen/numeric/*` artifacts from the SSOT -> all now 83. +2. Added `tools/check_catalog_count.py`: a CI invariant enforcing + `SSOT count == regenerated gen count == committed gen count`, and surfacing + any paper-count divergence as an explicit erratum reminder (not a silent + edit). +3. Added `.github/workflows/catalog-count-gate.yml` (CI-01): runs the invariant + on every PR touching the catalog, codegen, or gen artifacts; also fails if + committed gen artifacts are stale relative to a fresh codegen. + +## Recommended paper-side action + +Update arXiv:2606.09686 with a v2 or a published erratum changing all "84" +references to "83", citing this reconciliation. Until v2 is posted, the +abstract's "84" should be read as superseded by the canonical SSOT count of 83. +The conformance packs, SHA-256 anchors, and the phi^2 + 1/phi^2 = 3 anchor +vector are unaffected. diff --git a/conformance/integrity/fixtures/fixture_fp8_e5m2_posinf_roundtrip.json b/conformance/integrity/fixtures/fixture_fp8_e5m2_posinf_roundtrip.json new file mode 100644 index 000000000..057e5b354 --- /dev/null +++ b/conformance/integrity/fixtures/fixture_fp8_e5m2_posinf_roundtrip.json @@ -0,0 +1,19 @@ +{ + "fixture": "fp8_e5m2_pos_inf_special_roundtrip", + "kind": "special_value_row", + "expectation": "Check D2 MUST pass: +inf encodes and decodes to +inf; abs_error is a placeholder, not an arithmetic error; exempt from D and E.", + "pack_id": "fp8_e5m2", + "row": { + "name": "pos_inf", + "input_f64": "Inf", + "input_f64_hex": "0x7FF0000000000000", + "fp8_bits_hex": "0x7C", + "fp8_bits_int": 124, + "decoded_f64": "Inf", + "decoded_f64_hex": "0x7FF0000000000000", + "abs_error": 0.0, + "category": "inf" + }, + "severity": "none", + "note": "Same-class round-trip inf->inf. The v1 gate wrongly flagged this as a nan-mismatch (gate handling gap); v2 D2 validates it correctly." +} \ No newline at end of file diff --git a/conformance/integrity/fixtures/fixture_gf16_phi_mislabeled.json b/conformance/integrity/fixtures/fixture_gf16_phi_mislabeled.json new file mode 100644 index 000000000..110d3d3a9 --- /dev/null +++ b/conformance/integrity/fixtures/fixture_gf16_phi_mislabeled.json @@ -0,0 +1,20 @@ +{ + "fixture": "gf16_phi_mislabeled_abs_error", + "kind": "finite_value_row", + "expectation": "Check D MUST flag: stored abs_error 0.0 but abs(decoded-input) is nonzero.", + "pack_id": "gf16", + "row": { + "name": "phi", + "input_f64": 1.618033988749895, + "input_f64_hex": "0x3FF9E3779B97F4A8", + "gf16_bits_hex": "0x3F3C", + "gf16_bits_int": 16188, + "decoded_f64": 1.6171875, + "decoded_f64_hex": "0x3FF9E00000000000", + "abs_error": 0.0, + "category": "phi_anchor" + }, + "rederived_abs_error": 0.0008464887498949025, + "severity": "Risk", + "note": "gf16 decode of phi rounds to 1.6171875; the 8.46e-04 error was stored as 0.0. Anchor identity phi^2+1/phi^2=3 still holds in f64; only the per-row abs_error label is wrong." +} \ No newline at end of file diff --git a/conformance/vectors/INDEX_all_formats.json b/conformance/vectors/INDEX_all_formats.json index 5678b39ce..6fd3525b0 100644 --- a/conformance/vectors/INDEX_all_formats.json +++ b/conformance/vectors/INDEX_all_formats.json @@ -321,7 +321,7 @@ "file": "gf16_conformance_v0.json", "kind": "bitexact", "source": "hand-curated (pre-existing)", - "sha256": "7aea5b9e86ea71a54ae0c1601cea13e2d90d95fecaf2ae969eac1349cf7a2b42" + "sha256": "d1c0eb5bd66247b3c5db9a00a95e29cf4359653aec56f2f9e6827f96898d1509" }, { "id": "gf20", diff --git a/conformance/vectors/abs_error_allowlist.json b/conformance/vectors/abs_error_allowlist.json new file mode 100644 index 000000000..f6ff58101 --- /dev/null +++ b/conformance/vectors/abs_error_allowlist.json @@ -0,0 +1,54 @@ +{ + "schema": "wp18-abs-error-allowlist/v1", + "note": "Explicit, auditable allow-list of bit-exact-pack rows that carry a NONZERO abs_error for an honest, documented reason. Every entry is a row where the round-trip is CORRECT but the input is not exactly representable (not_on_grid) or saturates (overflow_to_inf). The gate fails on any nonzero abs_error NOT listed here. Never add an entry to silence a real error; each must be a genuinely-disclosed case with a fixture.", + "allow": [ + { + "pack_id": "bfloat16", + "row_name": "pos_0p1", + "reason": "not_on_grid", + "detail": "0.1 is not exactly representable in bf16; decoded_f64=0.10009765625, abs_error~9.77e-05 is the true nearest-representable error. Honest, expected." + }, + { + "pack_id": "gf16", + "row_name": "overflow_to_inf", + "reason": "overflow_to_inf", + "detail": "input 1e40 exceeds the gf16 dynamic range and saturates to +inf; abs_error=inf is the honest record of an out-of-range input, not an encode/decode bug." + }, + { + "pack_id": "gf16", + "row_name": "phi", + "reason": "not_on_grid", + "detail": "phi is not exactly representable on the gf16 grid; round-trip is correct, abs_error=0.000846489 is the true nearest-representable error (WP-18 F-1 correction)." + }, + { + "pack_id": "gf16", + "row_name": "inv_phi", + "reason": "not_on_grid", + "detail": "inv_phi is not exactly representable on the gf16 grid; round-trip is correct, abs_error=0.000130074 is the true nearest-representable error (WP-18 F-1 correction)." + }, + { + "pack_id": "gf16", + "row_name": "phi_squared", + "reason": "not_on_grid", + "detail": "phi_squared is not exactly representable on the gf16 grid; round-trip is correct, abs_error=0.000846489 is the true nearest-representable error (WP-18 F-1 correction)." + }, + { + "pack_id": "gf16", + "row_name": "inv_phi_sq", + "reason": "not_on_grid", + "detail": "inv_phi_sq is not exactly representable on the gf16 grid; round-trip is correct, abs_error=0.000130074 is the true nearest-representable error (WP-18 F-1 correction)." + }, + { + "pack_id": "gf16", + "row_name": "e", + "reason": "not_on_grid", + "detail": "e is not exactly representable on the gf16 grid; round-trip is correct, abs_error=0.000468172 is the true nearest-representable error (WP-18 F-1 correction)." + }, + { + "pack_id": "gf16", + "row_name": "pi", + "reason": "not_on_grid", + "detail": "pi is not exactly representable on the gf16 grid; round-trip is correct, abs_error=0.000967654 is the true nearest-representable error (WP-18 F-1 correction)." + } + ] +} \ No newline at end of file diff --git a/conformance/vectors/gf16_conformance_v0.json b/conformance/vectors/gf16_conformance_v0.json index 6c47959e1..2eeb0f82b 100644 --- a/conformance/vectors/gf16_conformance_v0.json +++ b/conformance/vectors/gf16_conformance_v0.json @@ -87,7 +87,7 @@ "gf16_bits_int": 16188, "decoded_f64": 1.6171875, "decoded_f64_hex": "0x3FF9E00000000000", - "abs_error": 0.0, + "abs_error": 0.0008464887498949025, "category": "phi_anchor" }, { @@ -98,7 +98,7 @@ "gf16_bits_int": 15481, "decoded_f64": 0.6181640625, "decoded_f64_hex": "0x3FE3C80000000000", - "abs_error": 0.0, + "abs_error": 0.0001300737501052085, "category": "phi_anchor" }, { @@ -109,7 +109,7 @@ "gf16_bits_int": 16542, "decoded_f64": 2.6171875, "decoded_f64_hex": "0x4004F00000000000", - "abs_error": 0.0, + "abs_error": 0.0008464887498949025, "category": "phi_anchor" }, { @@ -120,7 +120,7 @@ "gf16_bits_int": 15118, "decoded_f64": 0.3818359375, "decoded_f64_hex": "0x3FD8700000000000", - "abs_error": 0.0, + "abs_error": 0.00013007375010509747, "category": "phi_anchor" }, { @@ -230,7 +230,7 @@ "gf16_bits_int": 16568, "decoded_f64": 2.71875, "decoded_f64_hex": "0x4005C00000000000", - "abs_error": 0.0, + "abs_error": 0.0004681715409549092, "category": "normal" }, { @@ -241,7 +241,7 @@ "gf16_bits_int": 16676, "decoded_f64": 3.140625, "decoded_f64_hex": "0x4009200000000000", - "abs_error": 0.0, + "abs_error": 0.000967653589793116, "category": "normal" } ] diff --git a/docs/KAPPA_FP8_E4M3.md b/docs/KAPPA_FP8_E4M3.md new file mode 100644 index 000000000..e585230b3 --- /dev/null +++ b/docs/KAPPA_FP8_E4M3.md @@ -0,0 +1,73 @@ +# Quantifying the FP8 E4M3 Overflow Divergence with P3109 kappa + +**Status:** Draft. Implementor reference. No compliance claim, no superiority +claim. This document expresses a *known, documented* cross-vendor divergence in +the vocabulary the IEEE P3109 working group introduced for exactly this purpose. + +**Anchor:** kappa-approximation is defined in +[arXiv:2606.04028](https://arxiv.org/abs/2606.04028) (Fitzgibbon, Wintersteiger, +Sarnoff, "Novel Aspects of IEEE SA P3109 Arithmetic Formats for Machine +Learning", 1 Jun 2026): a scale-invariant accuracy measure, akin to units in the +last place but precisely defined for all operations, by which a vendor describes +an **approximate** implementation. An exact implementation must always be +provided; kappa describes how far an approximate one may deviate. + +--- + +## The divergence (already documented, now measured) + +For FP8 E4M3 (S1E4M3, bias 7, max finite 448.0 at 0x7E), the encode of an +overflowing input such as `1000.0` is **spec-permissible in two ways** under +OCP MX v1.0: + +| Convention | Behavior on encode(1000.0) | Result code | Decoded value | +|---|---|---|---| +| tt-metal / AMD | saturate to max finite | 0x7E | 448.0 | +| JAX / TPU (`ml_dtypes`) | overflow to NaN | 0x7F | NaN | + +Both are spec-compliant. The divergence silently changes model behavior when a +tensor is ported across vendors -- the exact "silent divergence" our conformance +packs exist to catch. + +## Expressing it as a kappa-approximation + +Treat the saturation path as an approximate implementation of the real value +`x = 1000.0`, and measure its deviation. + +- Max finite (decode 0x7E) = **448.0** (bit-exact). +- ULP at the top binade (exponent e_max = 15 - 7 = 8, 3 mantissa bits): + `2^(8-3) = 32.0`. +- Saturation absolute error vs the true real: `|448.0 - 1000.0| = 552.0`. +- In ULP at the top binade: `552.0 / 32.0 = 17.25 ULP`. +- **kappa (scale-invariant, relative):** `|approx - true| / |true| + = 552.0 / 1000.0 = 0.552`. + +For the JAX/TPU path, the implementation **declines to approximate**: it signals +NaN rather than returning a finite value. In kappa terms this is **undefined** +(no finite approximation is offered). This is the honest, precise distinction: + +| Path | P3109 Domain reading | kappa(encode 1000.0) | +|---|---|---| +| tt-metal / AMD saturate-to-max | Finite (saturating) | **0.552** (finite, large) | +| JAX / TPU overflow-to-NaN | Extended (signals NaN) | **undefined** (refuses to approximate) | + +## Why this is the right framing (and what we do NOT claim) + +- We do **not** claim either vendor is correct. Both are OCP-permissible. The + kappa value simply makes the *cost* of the saturation choice explicit and + scale-invariant, in the WG's own measure. +- We do **not** claim to define or own kappa. It is the P3109 WG's construct + (arXiv:2606.04028). We are applying it to a divergence we already ship as a + conformance vector, so the divergence becomes a *number a vendor can act on* + rather than a footnote. +- The bit-exact codes (0x7E saturate, 0x7F NaN) and the decoded values are the + Verified part. The kappa value 0.552 is a derived, reproducible measure + (`kappa_calc.py`), not an accuracy claim about any model. + +## Hook into the conformance pack + +The `fp8_e4m3` conformance pack already carries the 1000.0 row with both the +0x7E (ours) and the documented 0x7F (JAX/TPU) interpretation. This document adds +the kappa value as the standards-native way to report that row's cross-vendor +cost. A future pack field `kappa_overflow` can carry this per-vendor so the +divergence is machine-readable, not just prose. diff --git a/docs/NOW.md b/docs/NOW.md index 92933842d..50e4d249e 100644 --- a/docs/NOW.md +++ b/docs/NOW.md @@ -2,6 +2,15 @@ Last updated: 2026-06-14 +## conformance: catalog-count CI gate + WP-18 corpus-integrity gate + P3109 4-param cross-walk (this PR) + +- **WHAT**: the count-drift follow-up to #1064. The #1064 parser fix landed; this PR adds the CI guards that keep the catalog count and the conformance corpus honest going forward, plus the P3109 re-anchor docs. Per the repo constitution (L2 GENERATION), gen/ is DERIVED and is NOT hand-committed here; the count-gate regenerates fresh against the SSOT instead of diffing committed gen/. +- **COUNT GATE** (NEW): `tools/check_catalog_count.py` + `.github/workflows/catalog-count-gate.yml` fail any PR where SSOT != fresh-codegen. Paper-declared 84 emits a non-fatal WARN pointing at the errata. Live: SSOT == fresh regen == 83 (canonical), exit 0. +- **INTEGRITY GATE** (NEW): `tools/wp18_conformance_gate.py` + `tools/wp18_selftest_gate.py` + `.github/workflows/conformance-integrity-gate.yml` lock the conformance vector corpus against silent drift and undisclosed error (pack-set==SSOT, INDEX recount, per-pack SHA, abs_error re-derivation, special-value round-trip, honesty allow-list). 15/15 self-test gates the gate. F-1 fix: 6 gf16 rows (phi, inv_phi, phi_squared, inv_phi_sq, e, pi) stored abs_error=0.0 while decoding off-grid; abs_error corrected to measured values and allow-listed as not_on_grid; anchor phi^2 + 1/phi^2 = 3 still exact in f64; verdict CLEAN. +- **ERRATA** (NEW): `ERRATA_2026-06-14.md` reconciles arXiv:2606.09686 (declares 84) with the canonical SSOT 83. Path (a): SSOT is canonical, paper is errata-bound. The 84/83 delta is the MXFP6 E2M3/E3M2 split recorded in the paper but collapsed to one row in the SSOT. +- **P3109 docs** (NEW): `docs/P3109_CROSSWALK_4PARAM.md` re-anchors the cross-walk to the WG 4-parameter family (K bitwidth, P precision incl. implicit bit, Sigma signedness, Delta domain Finite/Extended) per arXiv:2606.04028; `docs/KAPPA_FP8_E4M3.md` records kappa=0.552 for the FP8 E4M3 saturate path (17.25 ULP top-binade) as a vendor-divergence measure; `docs/POSITIONING_CONFORMANCE_LAYER.md` positions t27 as the conformance/registry layer (Layer 3) under P3109-WG (Layer 1) and FLoPS/Imandra semantic verification (Layer 2), with takum (Hunhold, arXiv:2412.20273) cited openly as a breadth counterexample. +- L6 gf16 SSOT untouched. Specs SSOT `specs/numeric/formats_catalog.t27` untouched (already 83). gen/ NOT modified (L2-compliant). Docs + CI + corpus-label fixes only; no `coq/`/`bootstrap/` edits; no new `*.sh`. ASCII-only; honest abs_error preserved; no quality claim added. +- Closes #1079. ## ci-fix-bypass-echo -- fix no-op bypass step shell parse error (Closes #1031) - **WHERE** (CI only): `.github/workflows/l1-traceability.yml`, `.github/workflows/issue-gate.yml`, `.github/workflows/now-sync-gate.yml`. The trusted-bot no-op step used an inline `run: echo "... (Closes #1031)."`; the `#` opened a YAML/shell comment that truncated the command and the unbalanced parenthesis made the shell exit 2, so the gate job concluded `failure` for bots even though IS_BOT was correctly true and all substantive steps skipped. Switched to a block scalar (`run: |`) with a plain ASCII echo (no `#`, no parentheses). No source/specs/codegen/conformance/`gen/` touched. diff --git a/docs/P3109_CROSSWALK_4PARAM.md b/docs/P3109_CROSSWALK_4PARAM.md new file mode 100644 index 000000000..7061e94fa --- /dev/null +++ b/docs/P3109_CROSSWALK_4PARAM.md @@ -0,0 +1,93 @@ +# P3109 Cross-Walk (4-Parameter Re-Anchored) + +**Status:** Draft. Implementor cross-reference only. This document does NOT +assert P3109 compliance. Where our decode semantics disagree with the P3109 +formal model, we defer to FLoPS (arXiv:2602.15965, Lean formalization) and to +the IEEE P3109 working group's own definitions. + +**Re-anchored:** 2026-06-14. Supersedes any cross-walk labeled "P3109 v3.2.0". +The public P3109 anchor is the WG interim report (v0.9.1, 2024-10-29) **plus** +the working group's own 4-parameter redefinition in +[arXiv:2606.04028](https://arxiv.org/abs/2606.04028) ("Novel Aspects of IEEE SA +P3109 Arithmetic Formats for Machine Learning", Fitzgibbon, Wintersteiger, +Sarnoff, 1 Jun 2026). We map to THAT model, in the WG's own vocabulary. + +--- + +## The P3109 4-parameter model (their language) + +A P3109 format is defined by four parameters (per arXiv:2606.04028 and FLoPS): + +| Param | Symbol | Meaning | +|---|---|---| +| Bitwidth | K | total stored bits | +| Precision | P | significand bits **including the implicit leading bit** | +| Signedness | Sigma | Signed or Unsigned | +| Domain | Delta | Finite or Extended (Extended = +/-Inf and NaN present) | + +8-bit instances are named `binary8pP` for P in 1..7. For a `binaryK pP` format: +exponent field width `w = K - P` (signed), significand stored bits `= P - 1`. + +Two P3109 design rules relevant to the mapping: +- Single NaN (no payload), single zero (no -0). +- Domain controls special values: `Extended` carries +/-Inf and one NaN; + `Finite` carries neither. (Saturation handling is an operation/rounding-mode + property, not a format parameter.) + +--- + +## Our representative rows -> P3109 (K, P, Sigma, Delta) + +Precision P = stored mantissa bits (`m`) + 1 implicit bit. Sigma = Signed for +every row below (all our FP rows carry a sign bit, `s=1`). Delta is determined +by whether the format defines Inf/NaN. + +| Our id | bits K | s | e | m | P = m+1 | P3109 tuple | P3109 name | Match quality | +|---|---|---|---|---|---|---|---|---| +| `fp8_e4m3` | 8 | 1 | 4 | 3 | 4 | (K=8, P=4, Signed, Finite\*) | binary8p4 | element match; domain/overflow gap (see kappa) | +| `fp8_e5m2` | 8 | 1 | 5 | 2 | 3 | (K=8, P=3, Signed, Extended) | binary8p3 | element + domain match (true Inf/NaN) | +| `mxfp4` (element) | 4 | 1 | 2 | 1 | 2 | (K=4, P=2, Signed, Finite) | binary4p2 | element match; block scale is orthogonal to P3109 | +| `binary16` (fp16) | 16 | 1 | 5 | 10 | 11 | (K=16, P=11, Signed, Extended) | binary16p11 | IEEE 754 half; P3109 conversion target | +| `bf16` | 16 | 1 | 8 | 7 | 8 | (K=16, P=8, Signed, Extended) | binary16p8 | element match; P3109 conversion target | +| `gf16` (this work) | 16 | 1 | 6 | 9 | 10 | (K=16, P=10, Signed, Extended) | binary16p10 | structural match; GF bias differs (PHI_BIAS) | +| `e8m0` (block scale) | 8 | 0 | 8 | 0 | -- | scale-only; NOT a P3109 datum format | -- | Orthogonal (block scale layer) | + +\* **fp8_e4m3 domain note (the kappa hook).** Our `fp8_e4m3` follows the +tt-metal/AMD convention: no Inf, single NaN at 0x7F/0xFF, encode-overflow +**saturates** to max-finite 448.0 (0x7E). JAX/TPU `ml_dtypes` uses +overflow-to-NaN (0x7F). OCP MX v1.0 permits both. In P3109 4-parameter terms +this is a `Finite`-vs-near-`Extended` distinction combined with a saturation +choice. It is exactly the divergence P3109's **kappa-approximation** measure is +designed to quantify (see `KAPPA_FP8_E4M3.md`). + +### Orthogonal-by-design (no P3109 tuple) + +- **E8M0 block scale**: a shared exponent scale for a block of elements, not a + scalar datum format. P3109 defines block operations over a shared scale + uniformly, but the scale encoding itself is outside the binaryKpP datum + family. Listed as Orthogonal. +- **GF16 bias**: GF16's exponent bias is phi-derived (PHI_BIAS), not the P3109 + `2^(w-1)` / `2^(w-1)-1` convention. The bit *layout* maps cleanly to + binary16p10; the *value set* differs by bias. Marked structural match, value + set divergence -- defer to FLoPS-style value-set comparison, do not assert + equality. + +## Note on the P3109 bias convention (live, contested) + +The P3109 WG voted the exponent bias to `2^(w-1)-1` on 2025-04-28, then reversed +to `2^(w-1)` on 2025-07-07 (Overton, "Update Regarding the Exponent Bias", +2025-07-20). Any bias-sensitive row in this cross-walk (notably the FP8 rows) +should be read against whichever bias the consuming P3109 draft pins. We carry +both interpretations in the conformance packs and do not hard-code one as +"the" P3109 bias. + +## What we defer on (honesty wall) + +- **Decode semantics / rounding / projection:** defer to FLoPS (Lean) and the + WG. Our packs assert bit-exact encode/decode of the element layout, not the + full operation semantics P3109 specifies (Fast2Sum, ExtractScalar, FMA, + scaled add/mul, stochastic rounding). We are the conformance/registry layer, + not the proof layer. +- **The 4-bit and sub-8-bit P3109 names** (binary4pP) are our reading of the + parameterized family extended below 8 bits; the WG's public naming focuses on + binary8pP. Treat binary4p2 as an implementor label, not a WG-blessed name. diff --git a/docs/POSITIONING_CONFORMANCE_LAYER.md b/docs/POSITIONING_CONFORMANCE_LAYER.md new file mode 100644 index 000000000..eb3923dfb --- /dev/null +++ b/docs/POSITIONING_CONFORMANCE_LAYER.md @@ -0,0 +1,132 @@ +# Positioning: t27 as the Conformance / Registry Layer + +**Status:** Strategy note. Implementor-facing. No compliance claim, no +superiority claim. This document fixes WHERE the t27 numeric catalog sits in the +numeric-format ecosystem so that every info-drop, cross-walk, and pack speaks +from the same coordinates. + +**Re-anchored:** 2026-06-14 (companion to `P3109_CROSSWALK_4PARAM.md`, +`KAPPA_FP8_E4M3.md`, `ERRATA_2026-06-14.md`). + +--- + +## One sentence + +We are the **conformance / registry layer**: a vendor-neutral catalog of numeric +formats with bit-exact, honestly-erroring conformance vectors. We are **not** a +standards body, **not** a formal-proof engine, and **not** a competing format +family that claims to beat anyone on accuracy. + +## The layer stack (who does what) + +``` +LAYER 1 STANDARDS IEEE SA P3109 WG defines the format family + (defines) (arXiv:2606.04028, (K, P, Signedness, Domain), + interim report v0.9.1) bias, special values, + operation semantics + | + v defers-to / verifies-against +LAYER 2 FORMAL PROOF FLoPS (Rutgers/UCR, machine-checked Lean model + (verifies) arXiv:2602.15965); of P3109 semantics; finds + + Imandra P3109; fixes spec bugs; the source + Rutgers FP verification of truth for SEMANTICS + | + v we defer-to on semantics +LAYER 3 CONFORMANCE >>> t27 numeric catalog <<< bit-exact encode/decode + (measures + lists) (THIS WORK) vectors per format, with + 83-format SSOT, HONEST abs_error; a + 6 JSON packs, vendor-neutral REGISTRY of + Corona silicon oracle what each format's bits mean + | + v consumed-by +LAYER 4 IMPLEMENTORS tt-metal, ml_dtypes, use our vectors as a + (consume) PyTorch, IREE, vLLM, cross-vendor "does it fit" + llama.cpp, ONNX Runtime ruler when two stacks + disagree on the same format +``` + +Reading the stack: +- **Layer 1 decides what a format IS.** We never argue with the WG; we map to + their 4-parameter model in their own vocabulary (see `P3109_CROSSWALK_4PARAM.md`). +- **Layer 2 verifies the SEMANTICS** (rounding, projection, FMA, special-value + handling). Where our decode disagrees with FLoPS or Imandra, **they win** and + we annotate the divergence. We do not attempt to out-verify a Lean/Imandra + formalization -- that is their layer, and they are better at it. +- **Layer 3 is ours: a REGISTRY + a RULER.** Given a format, we publish exactly + which bit pattern decodes to which real value, and we state the error honestly + where a value is not representable. When P3109 leaves a choice open (e.g. FP8 + E4M3 saturate-vs-NaN on overflow), we do not pick a winner -- we **measure the + cost of each choice** in the WG's own kappa-approximation (see + `KAPPA_FP8_E4M3.md`). +- **Layer 4 consumes us.** The value to an implementor is: when two stacks + disagree on the same nominal format, our pack is the neutral arbiter that says + which bits mean what, with the divergence quantified. + +## Why we do NOT compete with Layer 2 + +Three concrete reasons, stated plainly so no drop ever drifts into a turf claim: + +1. **Different artifact.** FLoPS/Imandra produce machine-checked theorems about + operation semantics. We produce a table of bit patterns and decoded values. + A theorem and a vector table answer different questions ("is the rounding + rule sound?" vs "what does 0x7E decode to in this vendor's E4M3?"). +2. **We depend on them.** Our cross-walk explicitly defers to FLoPS where decode + semantics disagree. A registry that argued with the formalization it cites + would be incoherent. +3. **Standing.** A WG audience judges a superiority claim harder than any single + repo. The honest, useful posture -- "here is a cross-reference, please correct + it" -- is the only one that builds name at Layer 1/2. (See + `ruler-reputation-method` S1->S5 funnel.) + +## Breadth, not per-rung superiority (the takum rule) + +Our catalog earns its place through **breadth and toolchain coherence** across +~83 formats in 13 clusters -- **not** by claiming any single format beats a +competitor on accuracy. This is a hard rule, mirrored from the Corona ROM +governing sentence. + +- **takum** (Hunhold, + [arXiv:2412.20273](https://arxiv.org/abs/2412.20273), + [arXiv:2404.18603](https://arxiv.org/abs/2404.18603), + [arXiv:2504.21197](https://arxiv.org/abs/2504.21197)) is the **standing + counterexample**: at low precision takum is competitive with or better than + several of our own rungs, and reportedly meets-or-beats OFP8 at low precision. + We **ship takum un-suppressed in the catalog / Corona ROM**. We do not hide it, + down-weight it, or frame our ladder as superior to it. +- The "GoldenFloat ladder is better per-rung" claim (FL-002) stays **[Open + conjecture]** and is never asserted in any drop. What we assert is coverage + + bit-exact traceability across the whole family, takum included. + +This breadth posture is itself the moat: a registry is only trustworthy if it +lists formats that outperform its author's own. Listing takum honestly is the +credibility we trade on. + +## What this means operationally + +- **Info-drops** lead with a bit-exact fact + honest error, cite the relevant + layer above us (P3109 for the format definition, FLoPS for semantics), and ask + nothing. Never "our format is better"; always "here is the cross-vendor + divergence, measured." +- **Cross-walks** map to the P3109 4-parameter model and defer on semantics. +- **The kappa hook** is how we report an open WG choice as a number, in the WG's + own measure, without taking a side. +- **The catalog count** is whatever the live SSOT says (run the count, never + quote from memory -- see `ERRATA_2026-06-14.md` and the CI gate + `tools/check_catalog_count.py`). Today: 83. + +## What we explicitly do NOT claim (honesty wall) + +- We do not claim P3109 compliance -- only an implementor cross-reference. +- We do not claim our formats beat any competitor on any metric. +- We do not claim to verify operation semantics -- we defer to Layer 2. +- We do not claim "first" or "only" anything (banned-hype rule). +- We do not claim a count from memory -- the live SSOT is the only source. + +## References + +- IEEE P3109 4-parameter model: [arXiv:2606.04028](https://arxiv.org/abs/2606.04028) +- FLoPS Lean formalization: [arXiv:2602.15965](https://arxiv.org/abs/2602.15965) +- Imandra P3109: https://github.com/imandra-ai/ieee-p3109 +- takum: [arXiv:2412.20273](https://arxiv.org/abs/2412.20273) +- GoldenFloat preprint: arXiv:2606.05017 +- Catalog paper #3: arXiv:2606.09686 (count reconciled in `ERRATA_2026-06-14.md`) diff --git a/tools/check_catalog_count.py b/tools/check_catalog_count.py new file mode 100644 index 000000000..4994517f7 --- /dev/null +++ b/tools/check_catalog_count.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Catalog count invariant gate (CI-01 enforcement). + +Kills the count-drift class of bug. Enforces, in this order: + + 1. SSOT count == regenerated gen JSON count (HARD FAIL on mismatch) + 2. SSOT count == PAPER_DECLARED_COUNT (WARN -> errata required) + +The SSOT count is the single canonical number: the count of `// CATALOG:` +lines in specs/numeric/formats_catalog.t27. Codegen run fresh against the +SSOT must reproduce it exactly (this catches the parser-bug class that +silently dropped formula-bias rows). The paper count is declared here as a +constant; when it diverges, CI emits a loud errata reminder rather than +silently shipping divergent numbers. + +NOTE on gen/: per the repo constitution (L2 GENERATION), gen/ artifacts are +DERIVED and are never hand-committed in a PR. This gate therefore does NOT +compare against committed gen/ -- it regenerates into a temp dir and checks +that against the SSOT. The canonical number lives in the SSOT, full stop. + +Usage: + python3 tools/check_catalog_count.py + python3 tools/check_catalog_count.py --ssot PATH --tool PATH + +Exit codes: + 0 hard invariant holds (paper mismatch is WARN only by default) + 2 hard invariant violated (SSOT != fresh regen) + 3 paper count diverges AND --strict-paper passed +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +# The count claimed in arXiv:2606.09686 Table 1 abstract ("exactly 84"). +# This is the ASPIRATIONAL paper number. When SSOT diverges from it, an +# erratum is required (see ERRATA_2026-06-14.md). Do NOT silently edit this +# to match SSOT -- the whole point is to surface the divergence. +PAPER_DECLARED_COUNT = 84 +PAPER_ID = "arXiv:2606.09686" + +CATALOG_LINE = re.compile(r"//\s*CATALOG:") + + +def ssot_count(ssot: Path) -> int: + text = ssot.read_text(encoding="utf-8") + return sum(1 for line in text.splitlines() if CATALOG_LINE.search(line)) + + +def regen_count(ssot: Path, tool: Path) -> int: + """Run codegen into a temp dir and read its JSON count (independent path).""" + with tempfile.TemporaryDirectory() as td: + r = subprocess.run( + [sys.executable, str(tool), str(ssot), td], + capture_output=True, text=True, + ) + if r.returncode != 0: + print("codegen failed:\n" + r.stderr, file=sys.stderr) + sys.exit(2) + # surface any malformed-line warnings as hard signal + if "WARN: malformed" in r.stderr: + print("codegen dropped a CATALOG line (parser bug regressed):", + file=sys.stderr) + for ln in r.stderr.splitlines(): + if "WARN: malformed" in ln: + print(" " + ln, file=sys.stderr) + sys.exit(2) + gen = json.loads((Path(td) / "formats_catalog.json").read_text()) + return int(gen["count"]) + + +def main(argv: list[str]) -> int: + repo = Path(__file__).resolve().parent.parent + ap = argparse.ArgumentParser() + ap.add_argument("--ssot", + default=repo / "specs/numeric/formats_catalog.t27") + ap.add_argument("--tool", + default=repo / "tools/gen_formats_catalog.py") + ap.add_argument("--strict-paper", action="store_true", + help="treat paper-count divergence as a hard failure") + args = ap.parse_args(argv[1:]) + + ssot = Path(args.ssot) + tool = Path(args.tool) + + n_ssot = ssot_count(ssot) + n_regen = regen_count(ssot, tool) + + print(f"SSOT (// CATALOG: lines) = {n_ssot}") + print(f"regen (codegen fresh) = {n_regen}") + print(f"paper ({PAPER_ID} declared) = {PAPER_DECLARED_COUNT}") + + if n_ssot != n_regen: + print(f"FAIL: SSOT ({n_ssot}) != regen ({n_regen}) -- " + f"codegen drops/adds rows. Parser bug or SSOT malformed.", + file=sys.stderr) + return 2 + + if n_ssot != PAPER_DECLARED_COUNT: + msg = (f"WARN: SSOT ({n_ssot}) != paper count " + f"({PAPER_DECLARED_COUNT}). An erratum to {PAPER_ID} is " + f"required (see ERRATA_2026-06-14.md). Canonical live count " + f"is {n_ssot}.") + print(msg, file=sys.stderr) + if args.strict_paper: + return 3 + + print(f"OK: SSOT == fresh regen == {n_ssot} (canonical).") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools/wp18_conformance_gate.py b/tools/wp18_conformance_gate.py new file mode 100644 index 000000000..297ee886a --- /dev/null +++ b/tools/wp18_conformance_gate.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +WP-18 conformance-corpus integrity gate (stdlib only, ZERO local imports). + +Locks the t27 numeric conformance corpus against silent drift, the conformance-layer +analog of the catalog-count gate. Checks, as DISTINCT failures (CertifiedData v2 +"which surface drifted" discipline): + + A. pack-set name-set == SSOT format id-set (no missing / no extra) + B. INDEX total/bitexact/structural counts == recount from the pack files + C. per-pack SHA-256 in INDEX == file hash (manifest freshness) + D. FINITE-VALUE row arithmetic consistency: for every row carrying input_f64 + + decoded_f64 + abs_error whose values are finite, the stored abs_error must equal + abs(decoded_f64 - input_f64) within a tiny tolerance (re-derivation is the oracle; + metamorphic round-trip relation). + D2. SPECIAL-VALUE round-trip: rows encoding a special value (category in {inf, nan}, + i.e. an inf-code or nan-code) are validated by same-class round-trip instead of an + arithmetic magnitude: nan -> nan, +inf -> +inf, -inf -> -inf. Their abs_error is a + placeholder (nan for nan-codes; 0.0/inf for inf-codes) and is NOT an arithmetic + error, so it is exempt from D and E. + E. honesty allow-list: any FINITE NONZERO abs_error must appear on the explicit + allow-list with a machine-readable reason; an undisclosed finite nonzero error is + a LEAK. + +Exit 0 = CLEAN. Exit 2 = DRIFT or LEAK. Exit 3 = bad input. +NO capability/quality/benchmark metric is ever emitted. ASCII-only. Apache-2.0. + +Science backbone: Berkeley TestFloat/SoftFloat (independent re-derivation as oracle), +IeeeCC754, IBM FPgen vectors, Matula round-trip theorem (narrow-width decode-to-f64 is +exact; wide mantissas flagged), metamorphic testing (necessary-relation check), +ACM artifact review + CertifiedData v2 separate-surface hashing. See +RESEARCH_BIBLIOGRAPHY_WP18.md. + +Usage: + wp18_conformance_gate.py --ssot --vectors + [--allowlist ] [--json ] +""" +import argparse +import glob +import hashlib +import json +import math +import os +import re +import sys + +SCHEMA = "wp18-conformance-gate/v2" +# abs_error re-derivation tolerance: stored vs recomputed must agree this closely. +# Both are f64 subtractions of f64 inputs, so they should match to rounding noise. +REDERIVE_TOL = 1e-12 +# Row categories that denote a SPECIAL-VALUE code (inf / nan encoding), where +# abs(decoded - input) is not a meaningful arithmetic error and the correctness +# criterion is a same-class round-trip (inf->inf, nan->nan) instead. +SPECIAL_CATEGORIES = frozenset({"inf", "nan"}) + + +def _read_ssot_ids(ssot_path): + ids = [] + with open(ssot_path, "r", encoding="ascii") as fh: + for line in fh: + m = re.search(r"//\s*CATALOG:\s*id=(\S+)", line) + if m: + ids.append(m.group(1)) + return ids + + +def _sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def _to_float(v): + """Parse a stored numeric that may be a JSON number or 'Infinity'/'NaN' string.""" + if isinstance(v, bool): + return None + if isinstance(v, (int, float)): + return float(v) + if isinstance(v, str): + s = v.strip().lower() + if s in ("inf", "infinity", "+inf", "+infinity"): + return math.inf + if s in ("-inf", "-infinity"): + return -math.inf + if s in ("nan",): + return math.nan + return None + + +def _rows_of(pack): + return pack.get("vectors") or pack.get("rows") or [] + + +def _is_special_row(row, inp, dec): + """A special-value row encodes a non-finite value (inf/nan code). + + Identified by an explicit category tag in SPECIAL_CATEGORIES, or, defensively, + by a non-finite input or decoded value. Such a row's abs_error is a placeholder, + not an arithmetic magnitude. + """ + cat = (row.get("category") or "").strip().lower() + if cat in SPECIAL_CATEGORIES: + return True + if inp is not None and (math.isinf(inp) or math.isnan(inp)): + return True + if dec is not None and (math.isinf(dec) or math.isnan(dec)): + return True + return False + + +def run_gate(ssot_path, vectors_dir, allowlist_path=None): + report = { + "schema": SCHEMA, + "ssot": os.path.abspath(ssot_path), + "vectors_dir": os.path.abspath(vectors_dir), + "checks": {}, + "failures": [], + "capability_measured": False, # this gate measures integrity, never capability + } + + # ---- load INDEX ---- + index_path = os.path.join(vectors_dir, "INDEX_all_formats.json") + if not os.path.isfile(index_path): + report["failures"].append({"check": "input", "detail": "INDEX_all_formats.json not found"}) + return 3, report + try: + index = json.load(open(index_path, "r", encoding="ascii")) + except Exception as exc: # noqa: BLE001 + report["failures"].append({"check": "input", "detail": "INDEX parse error: %s" % exc}) + return 3, report + + # ---- load allow-list ---- + allow = {} + if allowlist_path and os.path.isfile(allowlist_path): + try: + al = json.load(open(allowlist_path, "r", encoding="ascii")) + for entry in al.get("allow", []): + allow[(entry["pack_id"], entry["row_name"])] = entry.get("reason", "") + except Exception as exc: # noqa: BLE001 + report["failures"].append({"check": "input", "detail": "allowlist parse error: %s" % exc}) + return 3, report + report["allowlist_entries"] = sorted("%s::%s" % k for k in allow) + + packs = index.get("packs", []) + pack_ids = set(p["id"] for p in packs) + + # ---- Check A: pack-set == SSOT id-set ---- + ssot_ids = set(_read_ssot_ids(ssot_path)) + if not ssot_ids: + report["failures"].append({"check": "input", "detail": "no SSOT CATALOG ids parsed"}) + return 3, report + missing = sorted(ssot_ids - pack_ids) + extra = sorted(pack_ids - ssot_ids) + report["checks"]["A_packset_equals_ssot"] = { + "ssot_count": len(ssot_ids), + "pack_count": len(pack_ids), + "missing_packs": missing, + "extra_packs": extra, + "ok": not missing and not extra, + } + if missing or extra: + report["failures"].append({"check": "A", "missing": missing, "extra": extra}) + + # ---- Check B: INDEX counts == recount ---- + be = st = 0 + file_missing = [] + for p in packs: + fn = os.path.join(vectors_dir, p["file"]) + if not os.path.isfile(fn): + file_missing.append(p["file"]) + continue + if p.get("kind") == "structural": + st += 1 + else: + be += 1 + claimed = (index.get("total_packs"), index.get("bitexact_packs"), index.get("structural_packs")) + recount = (len(packs), be, st) + report["checks"]["B_index_counts"] = { + "claimed_total_bitexact_structural": list(claimed), + "recount_total_bitexact_structural": list(recount), + "file_missing": file_missing, + "ok": claimed == recount and not file_missing, + } + if claimed != recount or file_missing: + report["failures"].append({"check": "B", "claimed": claimed, "recount": recount, "file_missing": file_missing}) + + # ---- Check C: SHA freshness ---- + sha_drift = [] + for p in packs: + fn = os.path.join(vectors_dir, p["file"]) + if not os.path.isfile(fn): + continue + want = p.get("sha256") + got = _sha256_file(fn) + if want and want != got: + sha_drift.append({"file": p["file"], "index_sha": want, "file_sha": got}) + report["checks"]["C_sha_freshness"] = {"drift_count": len(sha_drift), "drift": sha_drift[:10], "ok": not sha_drift} + if sha_drift: + report["failures"].append({"check": "C", "sha_drift": sha_drift[:10]}) + + # ---- Checks D + D2 + E: row re-derivation + special-value + honesty allow-list ---- + rederive_mismatch = [] + undisclosed_nonzero = [] + rows_checked = 0 + finite_rows = 0 + special_rows = 0 + special_mismatch = [] + nonzero_disclosed = [] + for p in packs: + if p.get("kind") == "structural": + continue # structural packs carry no bit-exact round-trip rows + fn = os.path.join(vectors_dir, p["file"]) + if not os.path.isfile(fn): + continue + try: + pack = json.load(open(fn, "r", encoding="ascii")) + except Exception as exc: # noqa: BLE001 + report["failures"].append({"check": "D", "file": p["file"], "detail": "parse error: %s" % exc}) + continue + for r in _rows_of(pack): + if not isinstance(r, dict): + continue + if "abs_error" not in r or "input_f64" not in r or "decoded_f64" not in r: + continue + ae_stored = _to_float(r.get("abs_error")) + inp = _to_float(r.get("input_f64")) + dec = _to_float(r.get("decoded_f64")) + if ae_stored is None or inp is None or dec is None: + continue + rows_checked += 1 + + # ---- D2: special-value rows (inf-code / nan-code) ---- + if _is_special_row(r, inp, dec): + special_rows += 1 + # Correctness = same-class round-trip. + if math.isnan(inp): + roundtrip_ok = (math.isnan(dec)) + elif math.isinf(inp): + roundtrip_ok = (math.isinf(dec) and (inp > 0) == (dec > 0)) + elif math.isinf(dec): + # finite input overflowed to inf (declared overflow_to_inf etc.): + # fall through to the finite-error honesty check below. + roundtrip_ok = None + else: + # tagged special but both finite -> treat as ordinary finite row + roundtrip_ok = None + if roundtrip_ok is False: + special_mismatch.append({"pack": p["id"], "row": r.get("name"), + "category": r.get("category"), + "input": str(inp), "decoded": str(dec), + "detail": "special-value round-trip broken"}) + if roundtrip_ok is not None: + continue # fully handled as a special-value row + + # ---- finite-value row (the meaningful arithmetic case) ---- + finite_rows += 1 + # D: re-derive abs_error from stored decoded vs input + rederived = abs(dec - inp) + if abs(rederived - ae_stored) > REDERIVE_TOL: + rederive_mismatch.append({"pack": p["id"], "row": r.get("name"), + "stored": ae_stored, "rederived": rederived}) + # E: honesty allow-list for any finite nonzero abs_error + is_nonzero = (math.isinf(ae_stored) or ae_stored != 0.0) + if is_nonzero: + key = (p["id"], r.get("name")) + if key in allow: + nonzero_disclosed.append({"pack": p["id"], "row": r.get("name"), + "abs_error": str(ae_stored), "reason": allow[key]}) + else: + undisclosed_nonzero.append({"pack": p["id"], "row": r.get("name"), + "abs_error": str(ae_stored)}) + report["checks"]["D_rederive_abs_error"] = { + "rows_checked": rows_checked, + "finite_rows": finite_rows, + "special_rows": special_rows, + "mismatch_count": len(rederive_mismatch), + "mismatch": rederive_mismatch[:20], + "ok": not rederive_mismatch, + } + report["checks"]["D2_special_value_roundtrip"] = { + "special_rows": special_rows, + "broken_count": len(special_mismatch), + "broken": special_mismatch[:10], + "ok": not special_mismatch, + } + report["checks"]["E_honesty_allowlist"] = { + "disclosed_nonzero": nonzero_disclosed, + "undisclosed_nonzero": undisclosed_nonzero, + "ok": not undisclosed_nonzero, + } + if rederive_mismatch: + report["failures"].append({"check": "D", "rederive_mismatch": rederive_mismatch[:20]}) + if special_mismatch: + report["failures"].append({"check": "D2", "special_mismatch": special_mismatch[:10]}) + if undisclosed_nonzero: + report["failures"].append({"check": "E", "undisclosed_nonzero": undisclosed_nonzero}) + + exit_code = 0 if not report["failures"] else 2 + report["verdict"] = "CLEAN" if exit_code == 0 else "DRIFT_OR_LEAK" + return exit_code, report + + +def main(argv=None): + ap = argparse.ArgumentParser(description="WP-18 conformance-corpus integrity gate") + ap.add_argument("--ssot", required=True, help="path to specs/numeric/formats_catalog.t27") + ap.add_argument("--vectors", required=True, help="path to conformance/vectors directory") + ap.add_argument("--allowlist", default=None, help="path to abs_error_allowlist.json") + ap.add_argument("--json", default=None, help="optional path to write the JSON report") + args = ap.parse_args(argv) + + if not os.path.isfile(args.ssot): + sys.stderr.write("bad-input: SSOT not found: %s\n" % args.ssot) + return 3 + if not os.path.isdir(args.vectors): + sys.stderr.write("bad-input: vectors dir not found: %s\n" % args.vectors) + return 3 + + code, report = run_gate(args.ssot, args.vectors, args.allowlist) + text = json.dumps(report, indent=2, sort_keys=True) + if args.json: + with open(args.json, "w", encoding="ascii") as fh: + fh.write(text + "\n") + print(text) + sys.stderr.write("WP-18 verdict: %s (exit %d)\n" % (report.get("verdict", "BAD_INPUT"), code)) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/wp18_selftest_gate.py b/tools/wp18_selftest_gate.py new file mode 100644 index 000000000..9c7bd86ea --- /dev/null +++ b/tools/wp18_selftest_gate.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +""" +WP-18 self-test for wp18_conformance_gate.py (stdlib only). + +A gate that can only ever say CLEAN is worthless. This self-test demonstrates the gate +is FALSIFIABLE: for every check (A, B, C, D, D2, E) we plant exactly one defect in a +synthetic mini-corpus and assert the gate FAILS that specific check; we also assert the +clean corpus PASSES, that the verdict is DETERMINISTIC across repeated runs, and that +the gate NEVER emits a capability/benchmark metric. + +This is the metamorphic "mutation-kill" discipline: a test suite is only trustworthy if +each planted mutant is killed by at least one assertion. + +Exit 0 = all self-tests pass. Exit 1 = a self-test failed (gate is not falsifiable). +ASCII-only. Apache-2.0. +""" +import copy +import hashlib +import json +import os +import shutil +import tempfile + +import wp18_conformance_gate as G + + +SSOT_TEXT = "\n".join([ + "// CATALOG: id=fp8_e4m3", + "// CATALOG: id=fp8_e5m2", + "// CATALOG: id=gf16", +]) + "\n" + + +def _sha(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + h.update(fh.read()) + return h.hexdigest() + + +def _bitexact_pack(pid, rows): + return {"schema": "t27-conformance/v0", "format": pid, "n_vectors": len(rows), "vectors": rows} + + +def _structural_pack(pid): + return {"schema": "t27-conformance/v0", "format": pid, "kind": "structural", "n_vectors": 0, "vectors": []} + + +def build_clean_corpus(root): + """A small, fully-consistent corpus: 2 bit-exact packs + 1 structural pack.""" + vec = os.path.join(root, "vectors") + os.makedirs(vec, exist_ok=True) + + ssot_path = os.path.join(root, "formats_catalog.t27") + with open(ssot_path, "w", encoding="ascii") as fh: + fh.write(SSOT_TEXT) + + # fp8_e4m3: one exact finite row + one nan-code row + one inf-code row + e4m3 = _bitexact_pack("fp8_e4m3", [ + {"name": "pos_1p0", "input_f64": 1.0, "decoded_f64": 1.0, "abs_error": 0.0, "category": "normal"}, + {"name": "nan_code", "input_f64": "nan", "decoded_f64": "nan", "abs_error": "nan", "category": "nan"}, + ]) + # fp8_e5m2: one exact row + one inf-code row (inf -> inf, abs_error placeholder 0.0) + e5m2 = _bitexact_pack("fp8_e5m2", [ + {"name": "pos_2p0", "input_f64": 2.0, "decoded_f64": 2.0, "abs_error": 0.0, "category": "normal"}, + {"name": "pos_inf", "input_f64": "inf", "decoded_f64": "inf", "abs_error": 0.0, "category": "inf"}, + ]) + # gf16 declared structural here (no bit-exact rows) to exercise the structural path + gf16 = _structural_pack("gf16") + + files = { + "fp8_e4m3_conformance_v0.json": e4m3, + "fp8_e5m2_conformance_v0.json": e5m2, + "gf16_conformance_v0.json": gf16, + } + for fn, obj in files.items(): + with open(os.path.join(vec, fn), "w", encoding="ascii") as fh: + json.dump(obj, fh) + + packs = [ + {"id": "fp8_e4m3", "file": "fp8_e4m3_conformance_v0.json", "kind": "bitexact", + "sha256": _sha(os.path.join(vec, "fp8_e4m3_conformance_v0.json"))}, + {"id": "fp8_e5m2", "file": "fp8_e5m2_conformance_v0.json", "kind": "bitexact", + "sha256": _sha(os.path.join(vec, "fp8_e5m2_conformance_v0.json"))}, + {"id": "gf16", "file": "gf16_conformance_v0.json", "kind": "structural", + "sha256": _sha(os.path.join(vec, "gf16_conformance_v0.json"))}, + ] + index = {"total_packs": 3, "bitexact_packs": 2, "structural_packs": 1, "packs": packs} + with open(os.path.join(vec, "INDEX_all_formats.json"), "w", encoding="ascii") as fh: + json.dump(index, fh) + + allow_path = os.path.join(root, "allowlist.json") + with open(allow_path, "w", encoding="ascii") as fh: + json.dump({"allow": []}, fh) + + return ssot_path, vec, allow_path + + +def _reindex_sha(vec): + """Recompute INDEX sha256 entries to match files on disk (use after editing files).""" + ipath = os.path.join(vec, "INDEX_all_formats.json") + idx = json.load(open(ipath)) + for p in idx["packs"]: + p["sha256"] = _sha(os.path.join(vec, p["file"])) + with open(ipath, "w", encoding="ascii") as fh: + json.dump(idx, fh) + + +def _edit_index(vec, fn): + ipath = os.path.join(vec, "INDEX_all_formats.json") + idx = json.load(open(ipath)) + fn(idx) + with open(ipath, "w", encoding="ascii") as fh: + json.dump(idx, fh) + + +def _edit_pack(vec, fname, fn): + fpath = os.path.join(vec, fname) + pack = json.load(open(fpath)) + fn(pack) + with open(fpath, "w", encoding="ascii") as fh: + json.dump(pack, fh) + + +def main(): + results = [] + + def check(name, ok): + results.append((name, bool(ok))) + + # ---------- T0: clean corpus PASSES ---------- + root = tempfile.mkdtemp(prefix="wp18st_clean_") + try: + ssot, vec, allow = build_clean_corpus(root) + code, rep = G.run_gate(ssot, vec, allow) + check("T0_clean_passes (exit 0 + verdict CLEAN)", code == 0 and rep["verdict"] == "CLEAN") + check("T0_no_capability_metric", rep.get("capability_measured") is False) + # no key anywhere that looks like a benchmark/score metric + blob = json.dumps(rep).lower() + forbidden = ["pass@", "compile@", "accuracy", "score", "benchmark", "throughput"] + check("T0_no_fabricated_metric_strings", not any(t in blob for t in forbidden)) + # determinism: run twice, byte-identical report (minus volatile abspaths already fixed) + _, rep2 = G.run_gate(ssot, vec, allow) + check("T0_deterministic", json.dumps(rep, sort_keys=True) == json.dumps(rep2, sort_keys=True)) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TA: Check A fails on extra pack (SSOT/pack mismatch) ---------- + root = tempfile.mkdtemp(prefix="wp18st_A_") + try: + ssot, vec, allow = build_clean_corpus(root) + # add an extra pack id to INDEX not present in SSOT + def add_extra(idx): + idx["packs"].append({"id": "ghost_fmt", "file": "fp8_e4m3_conformance_v0.json", + "kind": "bitexact", "sha256": "x"}) + idx["total_packs"] = 4 + idx["bitexact_packs"] = 3 + _edit_index(vec, add_extra) + code, rep = G.run_gate(ssot, vec, allow) + check("TA_extra_pack_fails_A", code == 2 and rep["checks"]["A_packset_equals_ssot"]["ok"] is False + and "ghost_fmt" in rep["checks"]["A_packset_equals_ssot"]["extra_packs"]) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TA2: Check A fails on missing pack ---------- + root = tempfile.mkdtemp(prefix="wp18st_A2_") + try: + ssot, vec, allow = build_clean_corpus(root) + # drop fp8_e5m2 from INDEX (still in SSOT) -> missing + def drop_one(idx): + idx["packs"] = [p for p in idx["packs"] if p["id"] != "fp8_e5m2"] + idx["total_packs"] = 2 + idx["bitexact_packs"] = 1 + _edit_index(vec, drop_one) + code, rep = G.run_gate(ssot, vec, allow) + check("TA2_missing_pack_fails_A", rep["checks"]["A_packset_equals_ssot"]["ok"] is False + and "fp8_e5m2" in rep["checks"]["A_packset_equals_ssot"]["missing_packs"]) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TB: Check B fails on wrong INDEX count ---------- + root = tempfile.mkdtemp(prefix="wp18st_B_") + try: + ssot, vec, allow = build_clean_corpus(root) + def bad_count(idx): + idx["bitexact_packs"] = 99 # lie about the count, keep packs list intact + _edit_index(vec, bad_count) + code, rep = G.run_gate(ssot, vec, allow) + check("TB_wrong_count_fails_B", code == 2 and rep["checks"]["B_index_counts"]["ok"] is False) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TC: Check C fails on stale SHA ---------- + root = tempfile.mkdtemp(prefix="wp18st_C_") + try: + ssot, vec, allow = build_clean_corpus(root) + # mutate a pack file body WITHOUT updating its INDEX sha256 -> stale manifest + _edit_pack(vec, "fp8_e4m3_conformance_v0.json", + lambda p: p["vectors"][0].update({"input_f64": 1.0000001, "decoded_f64": 1.0000001})) + code, rep = G.run_gate(ssot, vec, allow) + check("TC_stale_sha_fails_C", code == 2 and rep["checks"]["C_sha_freshness"]["ok"] is False) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TD: Check D fails on corrupted decoded_f64 (mislabeled abs_error) ---------- + root = tempfile.mkdtemp(prefix="wp18st_D_") + try: + ssot, vec, allow = build_clean_corpus(root) + # make a finite row whose decoded differs from input but abs_error still 0.0 + _edit_pack(vec, "fp8_e4m3_conformance_v0.json", + lambda p: p["vectors"][0].update({"decoded_f64": 1.5, "abs_error": 0.0})) + _reindex_sha(vec) # keep SHA fresh so ONLY D fails + code, rep = G.run_gate(ssot, vec, allow) + d = rep["checks"]["D_rederive_abs_error"] + check("TD_mislabeled_abs_error_fails_D", code == 2 and d["ok"] is False and d["mismatch_count"] >= 1) + check("TD_isolates_only_D", rep["checks"]["A_packset_equals_ssot"]["ok"] + and rep["checks"]["B_index_counts"]["ok"] and rep["checks"]["C_sha_freshness"]["ok"]) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TD2: Check D2 fails on broken special-value round-trip ---------- + root = tempfile.mkdtemp(prefix="wp18st_D2_") + try: + ssot, vec, allow = build_clean_corpus(root) + # nan-code row whose decoded is NOT nan (round-trip broken) + _edit_pack(vec, "fp8_e4m3_conformance_v0.json", + lambda p: p["vectors"][1].update({"input_f64": "nan", "decoded_f64": 0.0, "category": "nan"})) + _reindex_sha(vec) + code, rep = G.run_gate(ssot, vec, allow) + d2 = rep["checks"]["D2_special_value_roundtrip"] + check("TD2_broken_special_roundtrip_fails_D2", code == 2 and d2["ok"] is False and d2["broken_count"] >= 1) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TE: Check E fails on undisclosed finite nonzero abs_error ---------- + root = tempfile.mkdtemp(prefix="wp18st_E_") + try: + ssot, vec, allow = build_clean_corpus(root) + # finite row with a HONEST nonzero abs_error (consistent w/ D) but NOT on allowlist + def leak(p): + p["vectors"][0].update({"input_f64": 0.1, "decoded_f64": 0.10009765625, + "abs_error": abs(0.10009765625 - 0.1), "category": "normal"}) + _edit_pack(vec, "fp8_e4m3_conformance_v0.json", leak) + _reindex_sha(vec) + code, rep = G.run_gate(ssot, vec, allow) + e = rep["checks"]["E_honesty_allowlist"] + check("TE_undisclosed_nonzero_fails_E", code == 2 and e["ok"] is False and len(e["undisclosed_nonzero"]) >= 1) + check("TE_D_stays_clean (nonzero is honest)", rep["checks"]["D_rederive_abs_error"]["ok"]) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- TE2: same row, now allow-listed -> E passes (allowlist actually works) ---------- + root = tempfile.mkdtemp(prefix="wp18st_E2_") + try: + ssot, vec, allow = build_clean_corpus(root) + def leak2(p): + p["vectors"][0].update({"input_f64": 0.1, "decoded_f64": 0.10009765625, + "abs_error": abs(0.10009765625 - 0.1), "category": "normal"}) + _edit_pack(vec, "fp8_e4m3_conformance_v0.json", leak2) + _reindex_sha(vec) + with open(allow, "w", encoding="ascii") as fh: + json.dump({"allow": [{"pack_id": "fp8_e4m3", "row_name": "pos_1p0", + "reason": "0.1 not on fp8_e4m3 grid; honest rounding"}]}, fh) + code, rep = G.run_gate(ssot, vec, allow) + check("TE2_allowlisted_passes_E", code == 0 and rep["checks"]["E_honesty_allowlist"]["ok"] + and len(rep["checks"]["E_honesty_allowlist"]["disclosed_nonzero"]) >= 1) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- T_input: missing INDEX -> exit 3 ---------- + root = tempfile.mkdtemp(prefix="wp18st_in_") + try: + ssot, vec, allow = build_clean_corpus(root) + os.remove(os.path.join(vec, "INDEX_all_formats.json")) + code, rep = G.run_gate(ssot, vec, allow) + check("T_input_missing_index_exit3", code == 3) + finally: + shutil.rmtree(root, ignore_errors=True) + + # ---------- report ---------- + passed = sum(1 for _, ok in results if ok) + total = len(results) + print("WP-18 gate self-test: %d/%d assertions passed" % (passed, total)) + for name, ok in results: + print(" [%s] %s" % ("PASS" if ok else "FAIL", name)) + return 0 if passed == total else 1 + + +if __name__ == "__main__": + import sys + sys.exit(main())