Skip to content

Commit e9699ee

Browse files
Ixon: lazy, zero-copy anon env load for the Aiur check path (#445)
`ix check --ixe` deserialized the whole env eagerly: the Rust FFI materialized every constant and the Lean side built an object graph for all of them up front, even to check one lemma. On mathlib.ixe (2.97 GB) that was ~104 GB peak RSS / ~159 s just to load. Make the Lean `Ixon.Env` constants lazy and load them without copying: - `LazyConstant` is now an offset window `(buf, off, len)` into a shared backing buffer (+ optional materialized cache for the build path), parsed on demand. Only the checked closure is ever materialized. - New `rs_de_env_lazy` FFI: `Env::parse_lazy_index` walks the env once (reusing the existing parser, so every metadata variant incl. `CallSite` is handled) and returns per-const `(addr, offset, len)` windows into the buffer Lean passed in, plus `name -> addr` and per-`Defn` reducibility hints. No constant body is parsed or copied at load time. `deEnvAnon` reconstructs an `Env` of byte-window `LazyConstant`s over that same buffer. - This is anon-aligned with the circuit: binder metadata (the `ExprMetaArena`) is parsed-and-discarded, since the typecheck circuit only consumes anonymous constants, blobs, and the per-`Defn` hint. The CLI's `name -> addr` map is kept so targets can still be named. - Pure-Lean `getEnv`/`deEnv`/`rsDeEnv` keep their full-metadata behavior for round-trip and the non-check loaders; only the check path (`CheckCmd`) switches to `deEnvAnon`. Consumers updated for the lazy const type: closure walk, IOBuffer ingress (ships `rawBytes` windows directly), shard partitioning, decompile, and the compile/commit env builders. Measured on mathlib.ixe (same file, same target): 104.5 GB -> 4.5 GB peak RSS, 159 s -> 13.5 s, and the lazy run additionally typechecks the target rather than only resolving its address. Nothing in circuit changes: same closure bytes, same in-circuit blake3 verification, same proof.
1 parent 187c4e5 commit e9699ee

12 files changed

Lines changed: 472 additions & 41 deletions

File tree

Ix/Cli/CheckCmd.lean

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,10 @@ def forEachClaim
183183
match ixePath with
184184
| some path =>
185185
let bytes ← IO.FS.readBinFile path
186-
let ixonEnv ← match Ixon.rsDeEnv bytes with
186+
-- Anon load: lazy zero-copy constants, binder metadata dropped. The Aiur
187+
-- check circuit consumes only anonymous constants, blobs, and per-Defn
188+
-- reducibility hints — mirrors the Rust kernel's `get_anon_mmap`.
189+
let ixonEnv ← match Ixon.deEnvAnon bytes with
187190
| .error e =>
188191
IO.eprintln s!"Failed to deserialize {path}: {e}"; return 1
189192
| .ok env => pure env
@@ -331,7 +334,8 @@ private def blockAddrOf (addr : Address) (c : Ixon.Constant) : Address :=
331334
def ownedConstsForBlocks (ixonEnv : Ixon.Env) (blocks : Array Address) : Array Address := Id.run do
332335
let blockSet : Std.HashSet Address := blocks.foldl (·.insert ·) {}
333336
let mut o : Array Address := #[]
334-
for (addr, c) in ixonEnv.consts do
337+
for (addr, lc) in ixonEnv.consts do
338+
let some c := lc.get? | continue
335339
if blockSet.contains (blockAddrOf addr c) then o := o.push addr
336340
return o
337341

@@ -348,7 +352,7 @@ def loadEnvAndShards (manifestPath ixePath : String) :
348352
IO (Except String (Ixon.Env × Array (Array Address))) := do
349353
match parseIxesAllShards (← IO.FS.readBinFile manifestPath) with
350354
| .error e => return .error s!"manifest parse failed: {e}"
351-
| .ok shards => match Ixon.rsDeEnv (← IO.FS.readBinFile ixePath) with
355+
| .ok shards => match Ixon.deEnvAnon (← IO.FS.readBinFile ixePath) with
352356
| .error e => return .error s!"deserialize {ixePath} failed: {e}"
353357
| .ok env => return .ok (env, shards)
354358

@@ -405,7 +409,8 @@ def shardsCover (ixonEnv : Ixon.Env) (shards : Array (Array Address)) : IO Bool
405409
-- assign every const to a shard via its block; count + detect unowned.
406410
let mut counts : Array Nat := Array.replicate shards.size 0
407411
let mut unowned : Nat := 0
408-
for (addr, c) in ixonEnv.consts do
412+
for (addr, lc) in ixonEnv.consts do
413+
let some c := lc.get? | continue
409414
match blockToShard.get? (blockAddrOf addr c) with
410415
| some k => counts := counts.modify k (· + 1) -- total: no-op if out of range
411416
| none => unowned := unowned + 1

Ix/Commit.lean

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ open Ixon (PutM runPut putConstant putExpr Comm)
2828
def mkCompileEnv (phases : Ix.CompileM.CompilePhases) : Ix.CompileM.CompileEnv :=
2929
{ env := phases.rawEnv
3030
, nameToNamed := phases.compileEnv.named
31-
, constants := phases.compileEnv.consts
31+
, constants := phases.compileEnv.consts.fold (init := {})
32+
fun m a lc => m.insert a (lc.get?.getD default)
3233
, blobs := phases.compileEnv.blobs
3334
, totalBytes := 0 }
3435

Ix/CompileM.lean

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1615,7 +1615,8 @@ def compileEnv (env : Ix.Environment) (blocks : Ix.CondensedBlocks) (dbg : Bool
16151615
let allBlobs := nameBlobs.fold (fun m k v => m.insert k v) compileEnv.blobs
16161616

16171617
let ixonEnv : Ixon.Env := {
1618-
consts := compileEnv.constants
1618+
consts := compileEnv.constants.fold (init := {})
1619+
fun m a c => m.insert a (Ixon.LazyConstant.ofConstant c)
16191620
named := compileEnv.nameToNamed
16201621
blobs := allBlobs
16211622
names := namesMap
@@ -1908,7 +1909,8 @@ def compileEnvParallel (env : Ix.Environment) (blocks : Ix.CondensedBlocks)
19081909
IO.println s!" [Lean Compile] Blobs: {blockBlobCount} from blocks, {nameBlobCount} from names, {overlapCount} overlap, {finalBlobCount} final"
19091910

19101911
let ixonEnv : Ixon.Env := {
1911-
consts := constants
1912+
consts := constants.fold (init := {})
1913+
fun m a c => m.insert a (Ixon.LazyConstant.ofConstant c)
19121914
named := nameToNamed
19131915
blobs := allBlobs
19141916
names := namesMap

Ix/DecompileM.lean

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -712,7 +712,7 @@ def decompileProjection (cnst : Constant) (cMeta : ConstantMeta)
712712
def decompileOne (env : DecompileEnv) (ixonEnv : Ixon.Env)
713713
(_ixName : Ix.Name) (named : Ixon.Named)
714714
: Except String (Array (Ix.Name × Ix.ConstantInfo)) :=
715-
match ixonEnv.consts.get? named.addr with
715+
match ixonEnv.getConst? named.addr with
716716
| none => .ok #[]
717717
| some cnst =>
718718
let m : DecompileM (Array (Ix.Name × Ix.ConstantInfo)) :=
@@ -730,17 +730,17 @@ def decompileOne (env : DecompileEnv) (ixonEnv : Ixon.Env)
730730
let info ← decompileRecursor rec cnst named.constMeta
731731
pure #[(info.getCnst.name, info)]
732732
| .dPrj proj =>
733-
match ixonEnv.consts.get? proj.block with
733+
match ixonEnv.getConst? proj.block with
734734
| some { info := .muts mutuals, sharing, refs, univs } =>
735735
decompileProjection cnst named.constMeta mutuals sharing refs univs
736736
| _ => pure #[]
737737
| .iPrj proj =>
738-
match ixonEnv.consts.get? proj.block with
738+
match ixonEnv.getConst? proj.block with
739739
| some { info := .muts mutuals, sharing, refs, univs } =>
740740
decompileProjection cnst named.constMeta mutuals sharing refs univs
741741
| _ => pure #[]
742742
| .rPrj proj =>
743-
match ixonEnv.consts.get? proj.block with
743+
match ixonEnv.getConst? proj.block with
744744
| some { info := .muts mutuals, sharing, refs, univs } =>
745745
decompileProjection cnst named.constMeta mutuals sharing refs univs
746746
| _ => pure #[]

Ix/IxVM/ClaimHarness.lean

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ partial def closureFrom (env : Ixon.Env) (target : Address) : Std.HashSet Addres
9797
worklist := worklist.pop
9898
if visited.contains addr then continue
9999
visited := visited.insert addr
100-
let some c := env.consts.get? addr | continue
100+
let some c := env.getConst? addr | continue
101101
for r in c.refs do worklist := worklist.push r
102102
match c.info with
103103
| .iPrj p | .cPrj p => worklist := worklist.push p.block
@@ -111,8 +111,9 @@ partial def closureFrom (env : Ixon.Env) (target : Address) : Std.HashSet Addres
111111
one entry per const, keyed by its index. Returns the buffer and the
112112
count `n` (which the Aiur entrypoint receives as input). -/
113113
def buildSerdeIOBuffer (ixonEnv : Ixon.Env) : Aiur.IOBuffer × Nat :=
114-
ixonEnv.consts.valuesIter.fold (init := (default, 0)) fun (ioBuffer, i) c =>
115-
let (_, bytes) := Ixon.Serialize.put c |>.run default
114+
ixonEnv.consts.valuesIter.fold (init := (default, 0)) fun (ioBuffer, i) lc =>
115+
-- The lazy entry already holds the serialized bytes; no re-serialization.
116+
let bytes := lc.rawBytes
116117
(ioBuffer.extend 0 #[.ofNat i] (bytes.data.map .ofUInt8), i + 1)
117118

118119
/-- Encode a `Lean.ReducibilityHints` as a single `G` per the convention
@@ -139,9 +140,11 @@ private def hintToG : Lean.ReducibilityHints → Aiur.G
139140
def addEntries (ixonEnv : Ixon.Env) (keep : Address → Bool)
140141
(ioBuffer : Aiur.IOBuffer) : Aiur.IOBuffer := Id.run do
141142
let mut ioBuffer := ioBuffer
142-
for (addr, c) in ixonEnv.consts do
143+
for (addr, lc) in ixonEnv.consts do
143144
if !keep addr then continue
144-
let (_, bytes) := Ixon.Serialize.put c |>.run default
145+
-- The kernel re-hashes these bytes against the key, so feed the exact
146+
-- serialized form the lazy entry holds — no materialization needed.
147+
let bytes := lc.rawBytes
145148
let key : Array Aiur.G := addr.hash.data.map .ofUInt8
146149
ioBuffer := ioBuffer.extend 0 key (bytes.data.map .ofUInt8)
147150
for (addr, rawBytes) in ixonEnv.blobs do

0 commit comments

Comments
 (0)