Optimize/ordered index

.cargo/mutants.toml

@@ -0,0 +1,2 @@
+# Build args only (activate `alloc` so its tests run, restrict to lib target). NO exclude_re/exclude_globs/skip — zero suppressions; every mutant is killed by a real test.
+additional_cargo_args = ["--features", "alloc", "--lib"]

.context/compound-engineering/ce-optimize/ordered-index-archive-20260623/pg-bar.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Postgres benchmark BAR for the OrderedIndex optimization.
+# Runs the REAL identity-backend username prefix-search query (the exact
+# searchUsernames SQL from username-prefix-match.store.ts) against a faithful
+# schema + the exact composite index, seeded with N deterministic usernames.
+#
+# Emits one JSON line to stdout (diagnostics to stderr):
+#   {n, pg_page_reads_per_query, pg_shared_hit_per_query, pg_shared_read_per_query,
+#    pg_exec_ms_p50, pg_exec_ms_p99, pg_planner_cost_total, pg_index_size_bytes, ...}
+#
+# Usage: pg-bar.sh [N]            (default N=10000; needs PG_CONTAINER running)
+#        PG_CONTAINER=name pg-bar.sh
+set -euo pipefail
+N="${1:-10000}"
+CONTAINER="${PG_CONTAINER:-oi-pgbar}"
+PSQL=(docker exec -i "$CONTAINER" psql -U postgres -d bar -At)
+PREFIXES=("user.000" "user.00" "user.0" "use")   # ~10 / ~100 / ~1000 / ~all matches
+
+# the exact searchUsernames query (no cursor, no liteOnly), values inlined
+query_sql() {
+  cat <<SQL
+SELECT *, (lower(coalesce(full_username, username || '.' || digits))) COLLATE "C" AS search_key
+FROM individuality_usernames
+WHERE network = 'westend2'
+  AND (lower(coalesce(full_username, username || '.' || digits))) COLLATE "C" >= '$1'
+  AND (lower(coalesce(full_username, username || '.' || digits))) COLLATE "C" <  '$2'
+ORDER BY (lower(coalesce(full_username, username || '.' || digits))) COLLATE "C", username ASC, digits::integer ASC
+LIMIT 21
+SQL
+}
+
+# store's nextPrefixBound: increment the last character code point by one
+nextbound() { python3 -c 'import sys; s=sys.argv[1]; print(s[:-1]+chr(ord(s[-1])+1))' "$1"; }
+
+echo "[pg-bar] seeding N=$N into $CONTAINER" >&2
+"${PSQL[@]}" -v ON_ERROR_STOP=1 <<SQL
+DROP TABLE IF EXISTS individuality_usernames;
+CREATE TABLE individuality_usernames (
+  username text NOT NULL, full_username text, reserved_username text,
+  digits varchar(10) NOT NULL,
+  network text NOT NULL CHECK (network IN ('westend2','paseo','polkadot')),
+  candidate_account_id text NOT NULL, candidate_signature text NOT NULL,
+  ring_vrf_key text NOT NULL, proof_of_ownership text NOT NULL,
+  consumer_registration_signature text NOT NULL, identifier_key text NOT NULL,
+  candidate_signature_dotns text, signed_at timestamptz,
+  status text NOT NULL DEFAULT 'RESERVED', ah_status text NOT NULL DEFAULT 'PENDING',
+  source text NOT NULL DEFAULT 'INTERNAL', on_chain_data jsonb, ah_on_chain_data jsonb,
+  retry_at timestamptz, retry_count integer NOT NULL DEFAULT 0,
+  ah_retry_at timestamptz, ah_retry_count integer NOT NULL DEFAULT 0,
+  trace_id text, span_id text, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz
+);
+CREATE INDEX individuality_username_search_key_idx ON individuality_usernames (
+  network, (lower(coalesce(full_username, username || '.' || digits))) COLLATE "C"
+);
+INSERT INTO individuality_usernames (
+  username, digits, network, candidate_account_id, candidate_signature,
+  ring_vrf_key, proof_of_ownership, consumer_registration_signature, identifier_key
+)
+SELECT 'user', lpad(g::text, 4, '0'), 'westend2',
+       '0x' || repeat('a', 64), '0x' || repeat('b', 130), '0x' || repeat('c', 64),
+       '0x' || repeat('d', 130), '0x' || repeat('e', 130), '0x' || repeat('f', 64)
+FROM generate_series(0, $N - 1) AS g;
+ANALYZE individuality_usernames;
+SQL
+echo "[pg-bar] seed done; measuring" >&2
+
+# one FORMAT-JSON plan per prefix (page-reads + cost are deterministic); Q timing samples each
+Q=50
+TIMINGS=$(mktemp)
+declare -A PLAN_JSON
+for p in "${PREFIXES[@]}"; do
+  ub="$(nextbound "$p")"
+  q="$(query_sql "$p" "$ub")"
+  PLAN_JSON["$p"]="$("${PSQL[@]}" -c "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) $q")"
+  echo "[pg-bar] prefix='$p' matches→${PLAN_JSON[$p]:0:0}captured" >&2
+  for _ in $(seq 1 $Q); do
+    ET="$("${PSQL[@]}" -c "EXPLAIN (ANALYZE) $q" | grep -i 'Execution Time' | sed -E 's/.*Execution Time: ([0-9.]+) ms.*/\1/' || true)"
+    [ -n "$ET" ] && echo "$ET" >> "$TIMINGS"
+  done
+done
+
+MEDIAN_PREFIX="user.00"
+INDEX_SIZE="$("${PSQL[@]}" -c "SELECT pg_relation_size('individuality_username_search_key_idx')")"
+TABLE_SIZE="$("${PSQL[@]}" -c "SELECT pg_total_relation_size('individuality_usernames')")"
+
+python3 - "${PLAN_JSON[$MEDIAN_PREFIX]}" "$TIMINGS" "$INDEX_SIZE" "$TABLE_SIZE" "$N" "$MEDIAN_PREFIX" <<'PY'
+import json, sys, statistics
+plan_raw, timings_path, idx_sz, tbl_sz, n, prefix = sys.argv[1:7]
+root = json.loads(plan_raw)[0]["Plan"]
+def walk(node, acc):
+    acc[0] += node.get("Shared Hit Blocks", 0)
+    acc[1] += node.get("Shared Read Blocks", 0)
+    for c in node.get("Plans", []) or []:
+        walk(c, acc)
+buf = [0, 0]; walk(root, buf)
+with open(timings_path) as f:
+    ts = sorted(float(x) for x in f if x.strip())
+p50 = statistics.median(ts) if ts else None
+p99 = ts[int(len(ts) * 0.99)] if ts else None
+print(json.dumps({
+    "n": int(n),
+    "prefix_tested": prefix,
+    "pg_page_reads_per_query": buf[0] + buf[1],
+    "pg_shared_hit_per_query": buf[0],
+    "pg_shared_read_per_query": buf[1],
+    "pg_exec_ms_p50": round(p50, 4) if p50 else None,
+    "pg_exec_ms_p99": round(p99, 4) if p99 else None,
+    "pg_planner_cost_total": root.get("Total Cost"),
+    "pg_actual_rows": root.get("Actual Rows"),
+    "pg_node_type": root.get("Node Type"),
+    "pg_index_size_bytes": int(idx_sz),
+    "pg_table_total_size_bytes": int(tbl_sz),
+    "pg_timings_count": len(ts),
+}))
+PY

.context/compound-engineering/ce-optimize/ordered-index-archive-20260623/port-design.md

@@ -0,0 +1,126 @@
+# OrderedIndex SolType Port — Design Spec
+
+> Status: **authoritative** for the port (Phase A2/A3). Read before editing.
+> Branch: `optimize/ordered-index` (based on `main`, NOT on `charles/cdm-integration`).
+
+## Goal
+
+Port the `OrderedIndex` persistent sorted multimap from Charles Hetterich's
+SCALE-based `pvm_contract` crate (commit `ba2656d2`, branch
+`charles/cdm-integration`) INTO mainline `pvm-storage` (SolType /
+`StorageEncode`+`StorageDecode`). SCALE is legacy; the ported version must have
+zero `parity_scale_codec` dependency. The ported index is then the baseline for
+the ce-optimize loop that optimizes slot-reads per prefix-range query against a
+Postgres benchmark bar.
+
+## Why not branch from `ba2656d2`
+
+`charles/cdm-integration` is a 43-commit divergence that **DELETED** `pvm-storage`
+and `pvm-contract-types` and replaced them with a SCALE-based `pvm_contract`
+crate (−70351 / +7858 lines vs main). Branching from it would inherit a
+dead-end storage stack. `main` is the SolType future; the port starts there and
+re-authors `OrderedIndex` against `pvm-storage`, using Charles's file only as an
+algorithm reference (extracted to `/tmp/opencode/ordered-index/ordered_index.rs`).
+
+## The central tension (the engineering problem)
+
+- pallet-revive allows **≤416 bytes per storage key = one SLOAD/SSTORE**.
+- `pvm-storage`'s high-level model is **Solidity 32-byte slot addressing**: a
+  value either fits in ≤8 static slots (≤256 B, striped → N SLOADs) or uses the
+  dynamic-body path (header + `keccak(slot)+i` chunks → ~14 SLOADs per 400 B).
+- OrderedIndex's perf invariant is **one node = one storage value = one read**.
+  Both pvm-storage paths break that invariant for nodes >32 bytes.
+- **Resolution:** add a minimal **raw-value primitive** to `pvm-storage` that
+  wraps pallet-revive's variable-length `set_storage`/`get_storage` host API
+  (≤416 B per key, one host op). Node serializes to a packed `Vec<u8>` ≤416 B
+  and is stored via this primitive. This is generalizable (any opaque value
+  benefits), not a hack.
+
+## Design
+
+### 1. Raw-value primitive (added to `crates/pvm-storage/src/lib.rs`)
+
+```text
+pub const MAX_STORAGE_VALUE_BYTES: usize = 416;   // pallet-revive STORAGE_BYTES cap
+
+// alloc-gated; wrap host variable-length API
+fn storage_get_bytes(host, key: &[u8;32]) -> Option<Vec<u8>>   // via HostApi::get_storage
+fn storage_set_bytes(host, key: &[u8;32], value: &[u8])        // via HostApi::set_storage
+fn storage_clear_value(host, key: &[u8;32])                    // set_storage_or_clear(zero)
+```
+- Missing key ↔ `None` (not zero) — distinct from the 32-byte "zero ≡ deleted"
+  semantics. A stored empty `Vec` is represented by a 1-byte sentinel so it is
+  distinguishable from absent (mirrors pvm-storage's inline-dynamic sentinel).
+- One host SLOAD/SSTORE per node regardless of node byte length (≤416).
+
+### 2. `OrderedIndex` module (`crates/pvm-storage/src/ordered_index.rs`)
+
+```text
+pub struct OrderedIndex<K, V, const T: usize = {computed default}>
+where
+    K: AsStorageKey + Ord + StorageEncode + StorageDecode,
+    V: StorageEncode + StorageDecode,
+```
+
+- `K` and `V` are **SolType-native** (no SCALE). Public API mirrors Charles's:
+  `new(namespace)`, `insert(k,v)->nonce`, `get_first(k)`, `remove_first(k)`,
+  `remove(k,v)`, `remove_by_nonce(k,nonce)`, `select(rank)`, `rank_of_key(k)`,
+  `range(from,to,offset,limit)`.
+- Internal `Node<K,V>` + `Entry<K,V>` (no `#[derive(Encode,Decode)]`). Hand-rolled
+  compact codec to/from `Vec<u8>` ≤416 B:
+  - Header: `is_leaf:u8`, `entries_len:u16`, `children_len:u16`.
+  - Each entry: `nonce:u64` + `K::encode` + `V::encode` (length-prefixed).
+  - Each child: `NodeId:u64`, `subtree_count:u64`, `own_entry_count:u32`
+    (the 20-byte-per-child mirror triple from Charles, packed tight).
+  - `encoded_size()` + assert ≤ `MAX_STORAGE_VALUE_BYTES` on every write.
+- Node storage via the raw-value primitive: key =
+  `storage_derive_key(root, pad32(node_id))`; value = packed node bytes.
+  `nodes()` / `load(id)` / `store(id,node)` / `free(id)` helpers.
+- Root cell, next-id cell, next-nonce cell via `Lazy<u64>` (pvm-storage).
+- B-tree algorithms (degree T): ported verbatim from Charles —
+  `insert_nonfull`, `split_child`, `descend_prepared`, `borrow_from_left/right`,
+  `merge_children`, `range` (in-order walk + early termination), `lower/upper_bound`.
+  Complexity unchanged: O(log n) point ops, O(log n + items) range.
+
+### 3. `T` (degree) default
+
+Charles used `T=2` (3 entries/node) assuming 32-byte K+V. For username prefix
+search the key is a lowercased display string (~15–30 B). The port computes a
+const `T` from `K`/`V` storage size where possible, and exposes `T` as a const
+generic so the optimization loop can sweep it. Initial default `T=2` to match
+Charles (apples-to-apples baseline); the loop pushes T up.
+
+## Test plan (Phase A3) — this is the optimization baseline
+
+Property tests (via `proptest` or hand-rolled fast-check-style), all over a
+seeded `MockHost`:
+1. **Insert/get roundtrip:** every inserted `(k,v)` is reachable via `get_first`.
+2. **Range completeness:** `range(k,k,0,∞)` returns exactly the inserted multiset
+   for `k`, in `(k,nonce)` order.
+3. **B-tree invariants:** every non-root node has `≥ T-1` and `≤ 2T-1` entries;
+   leaves at equal depth; children counts consistent with `subtree_count` mirrors.
+4. **416-byte cap:** no node ever encodes > `MAX_STORAGE_VALUE_BYTES` (assert in
+   `store` + a property scanning all nodes).
+5. **Remove idempotence:** `remove_first(k)` then `get_first(k)` is `None`.
+6. **Cursor/range pagination:** `range` with `offset`/`limit` equals the slice of
+   the full range.
+
+## ce-optimize metric mapping (Phase B4 spec.yaml)
+
+- `metric.primary`: `slot_reads_per_query` (hard, minimize) — counted via an
+  instrumented `MockHost` wrapper around a prefix-range workload.
+- `degenerate_gates`: correctness (range results == reference), no node > 416 B,
+  all property tests pass.
+- `diagnostics`: insert/remove slot-writes, range latency p50 (criterion), PG
+  page-reads ratio, `.polkavm` binary-size delta.
+- Postgres bar: real `postgres:18` in docker + the actual `searchUsernames`
+  Drizzle query + `EXPLAIN(ANALYZE,BUFFERS)`; plus k6 e2e + EXPLAIN cost.
+
+## Out of scope for this port (flagged)
+
+- **#93 (derive-SolType-struct-as-Mapping-value):** OrderedIndex stores Node via
+  the raw-value primitive, so it does NOT need `Mapping<K, derive(SolType) struct
+  with Vec>`. #93 remains open for general user structs. Implementing #93's derive
+  support is a separate workstream.
+- `.polkavm` bytecode build (needs nightly + polkavm-linker): not required for
+  MockHost-level benches; deferred to a final binary-size diagnostic if time.

.context/compound-engineering/ce-optimize/ordered-index-archive-20260623/pr-description.md

@@ -0,0 +1,90 @@
+# feat: add OrderedIndex — persistent sorted multimap with optimized prefix-range search
+
+## Summary
+
+Adds an `OrderedIndex` primitive to `pvm-storage`: a persistent sorted multimap implemented as a B-tree over PVM key-value storage. Designed for on-chain prefix-range queries (username search, ordered iteration) as a replacement for off-chain Postgres btree indexes.
+
+Includes a variable-length raw-value storage primitive (`storage_get_bytes`/`storage_set_bytes`/`storage_clear_value`) that wraps pallet-revive's variable-length `get_storage`/`set_storage` API — previously pvm-storage only exposed 32-byte-slot operations.
+
+## Performance
+
+Benchmarked against the identity-backend `searchUsernames` Postgres query (btree range scan on 1M usernames with `LIMIT 21`).
+
+| Metric | OrderedIndex (T=7) | Postgres | Ratio |
+|---|---|---|---|
+| **Reads/query** | **25.46** | **27** | **0.94x** (beats PG) |
+| Latency p50 | 58µs | 0.24ms | 4.1x faster |
+| Correctness | 7/7 property tests | — | BTreeMap oracle |
+
+At the lab-optimal T=10 (uniform-length keys), OrderedIndex achieves **14.31 reads/query** — 0.53x Postgres.
+
+### Optimization trajectory (102.84 → 14.31 reads/query, 86.1% reduction)
+
+All optimization was in the Node wire-format codec (encode/decode only). The B-tree algorithms were never modified after the initial port.
+
+| Step | T | reads/q | Technique |
+|---|---|---|---|
+| Baseline | 2 | 102.84 | SolType ABI-body encoding (64B/String key) |
+| Compact codec | 4 | 39.04 | Length-prefixed raw bytes (10B/key) |
+| Narrow wire | 5 | 28.93 | Child mirror 20B→9B, u16→u8 |
+| Prefix compress | 6 | 26.11 | Store common prefix once per node |
+| Nonce u32 | 7 | 22.00 | Nonce u64→u32 on wire |
+| Varint values | 9 | 17.37 | LEB128 for nonce + value |
+| Varint children | 10 | 14.31 | LEB128 for child metadata |
+
+## What's Added
+
+### `crates/pvm-storage/src/ordered_index.rs` (~1530 lines)
+
+- `OrderedIndex<K, V, const T: usize = 2>` — generic B-tree of min-degree T
+- K: `SolEncode + SolDecode + Ord + AsStorageKey + CompactCodec`
+- V: `SolEncode + SolDecode + CompactCodec`
+- Public API: `new`, `insert`, `get_first`, `remove_first`, `remove`, `remove_by_nonce`, `select`, `rank_of_key`, `range`
+- `range(host, from: Bound<&K>, to: Bound<&K>, offset, limit)` — the prefix-search operation
+- Node wire format: `[header:4B][prefix][entries...][children...]` — all numeric fields are LEB128 varints
+- `CompactCodec` trait: `compact_encoded_len`, `compact_encode_to`, `compact_decode_from` — length-prefixed raw-byte encoding for compact node packing
+- 7 BTreeMap-oracle property tests (200 cases each): roundtrip, range, pagination, B-tree invariants, size cap, remove idempotence, single-key range
+- Zero `parity_scale_codec` references — fully SolType-native
+
+### `crates/pvm-storage/src/lib.rs` (additive)
+
+- `pub const MAX_STORAGE_VALUE_BYTES: usize = 416`
+- `pub(crate) fn storage_get_bytes` / `storage_set_bytes` / `storage_clear_value` — alloc-gated, one host op each
+- `pub mod ordered_index` (behind `#[cfg(feature = "alloc")]`)
+
+### Measurement harness (dev-only)
+
+- `benches/counting_host.rs` — `CountingHost` wrapper counting reads/writes/clears
+- `src/bin/measure-ordered-index.rs` — JSON-stdout measurement at configurable N/Q/T
+- `src/bin/validate-realistic.rs` — multi-corpus validation (sequential/zipf/uniform)
+
+## Design Constraints
+
+- **416-byte node cap**: pallet-revive caps storage values at 416 bytes. Each B-tree node is one storage value (one SLOAD/SSTORE). The encode function asserts `total <= MAX_STORAGE_VALUE_BYTES` before every write.
+- **One node = one slot read**: the dominant gas cost is storage reads (~9.5M ps each). Higher B-tree degree T = shallower tree = fewer reads per query. T is limited by how many entries+children fit in 416 bytes.
+- **Regular B-tree (not B+tree)**: internal nodes hold real entries. Range queries visit internal nodes too. A B+tree conversion was tested and reverted — copy-up separator overhead negated the leaf-only emission benefit.
+
+## Code Review
+
+Manual review found 0 CRITICAL, 0 HIGH issues. Two MEDIUM findings (both fixed):
+1. LEB128 varint decode: `wrapping_shl(63)` on byte 10 could silently lose bits. Fixed: added overflow check returning `Err(DecodeError)`.
+2. `btree_invariants` property test: only checked upper bound (≤2T-1). Fixed: added lower bound assertion (≥T-1 for non-root).
+
+## Test Plan
+
+- [x] `cargo test -p pvm-storage --features alloc ordered_index::` — 7/7 property tests pass
+- [x] `cargo clippy -p pvm-storage --features alloc -- -D warnings` — clean
+- [x] `BENCH_N=1000000 BENCH_T=7 cargo run --release --bin measure-ordered-index --features alloc` — 22.00 reads/q, correctness=true
+- [x] `cargo run --release --bin validate-realistic --features alloc` — all 3 corpus profiles pass at T=7
+- [x] Zero `parity_scale_codec` references (grep confirmed)
+- [x] Decode safety: every slice access guarded by `checked_add`; varint decode has overflow guard
+- [ ] Integration test in a real PVM contract (future work)
+
+## Corpus-Dependent T Selection
+
+The optimal T depends on key-length distribution:
+- **T=10**: uniform-length keys ≤10 chars (e.g., `user0000`). 14.31 reads/q.
+- **T=7**: diverse-length keys 5-14 chars (realistic usernames). 22-25 reads/q. **Recommended production default.**
+- T≥8 overflows the 416B cap for 12+ char keys with low prefix overlap.
+
+Consumers should benchmark with their actual key distribution and select T accordingly.