Add STATE_BACKEND support for A/B benchmarking (#118)

* Add STATE_BACKEND support for A/B benchmarking fjall vs lmdb

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Pass STATE_BACKEND directly to JAM_FUZZ_DB (support full fjall/lmdb)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Bump conformance timeout to 35min for full-fjall backend

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: tomusdrw

.github/workflows/docker-imports-full.yml

@@ -7,6 +7,10 @@ on:
         description: 'typeberry version: `next` (default), `<semver>-<sha>` (pre-release), or `<semver>` (release)'
         required: false
         default: 'next'
+      backend:
+        description: 'State backend: fjall, lmdb, fjall-hybrid, lmdb-hybrid (default: version default)'
+        required: false
+        default: ''
   schedule:
     # 3:30 AM Europe/Warsaw (1:30 UTC, using UTC+2 summer time)
     - cron: '30 1 * * *'
@@ -60,6 +64,8 @@ jobs:
       - name: Run full-chain import test
         run: |
           npm exec tsx --test tests/docker-imports-full.test.ts
+        env:
+          STATE_BACKEND: ${{ github.event.inputs.backend || '' }}
 
       - name: Reap test containers and volumes
         if: always()

.github/workflows/docker-imports.yml

@@ -11,6 +11,10 @@ on:
         description: 'typeberry version: `next` (default), `<semver>-<sha>` (pre-release), or `<semver>` (release)'
         required: false
         default: 'next'
+      backend:
+        description: 'State backend: fjall, lmdb, fjall-hybrid, lmdb-hybrid (default: version default)'
+        required: false
+        default: ''
   repository_dispatch:
   schedule:
     # 1x/day (reduced from 5x) — staggered overnight to relieve the single shared self-hosted runner.
@@ -66,6 +70,8 @@ jobs:
       - name: Run Docker Imports test
         run: |
           npm exec tsx --test tests/docker-imports.test.ts
+        env:
+          STATE_BACKEND: ${{ github.event.inputs.backend || '' }}
 
       - name: Reap test containers and volumes
         if: always()

.github/workflows/minifuzz.yml

@@ -11,6 +11,10 @@ on:
         description: 'typeberry version: `next` (default), `<semver>-<sha>` (pre-release), or `<semver>` (release)'
         required: false
         default: 'next'
+      backend:
+        description: 'State backend: fjall, lmdb, fjall-hybrid, lmdb-hybrid (default: version default)'
+        required: false
+        default: ''
   repository_dispatch:
   schedule:
     # 2x/day (reduced from 5x) — staggered overnight to relieve the single shared self-hosted runner.
@@ -57,6 +61,8 @@ jobs:
       - name: Run Minifuzz ${{ matrix.test }} test
         run: |
           npm exec tsx --test tests/minifuzz/${{ matrix.test }}.test.ts
+        env:
+          STATE_BACKEND: ${{ github.event.inputs.backend || '' }}
 
       - name: Reap test containers and volumes
         if: always()

.github/workflows/npm-minifuzz.yml

@@ -11,6 +11,10 @@ on:
         description: 'typeberry version: `next` (default), `<semver>-<sha>` (pre-release), or `<semver>` (release)'
         required: false
         default: 'next'
+      backend:
+        description: 'State backend: fjall, lmdb, fjall-hybrid, lmdb-hybrid (default: version default)'
+        required: false
+        default: ''
   repository_dispatch:
   schedule:
     # 2x/day (reduced from 5x) — staggered overnight to relieve the single shared self-hosted runner.
@@ -56,3 +60,5 @@ jobs:
       - name: Run Minifuzz ${{ matrix.test }} test (npm image)
         run: |
           npm exec tsx --test tests/minifuzz/${{ matrix.test }}.test.ts
+        env:
+          STATE_BACKEND: ${{ github.event.inputs.backend || '' }}

.github/workflows/picofuzz-full.yml

@@ -7,6 +7,10 @@ on:
         description: 'typeberry version: `next` (default), `<semver>-<sha>` (pre-release), or `<semver>` (release)'
         required: false
         default: 'next'
+      backend:
+        description: 'State backend: fjall, lmdb, fjall-hybrid, lmdb-hybrid (default: version default)'
+        required: false
+        default: ''
   schedule:
     # 4:30 AM Europe/Warsaw (2:30 UTC, using UTC+2 summer time)
     - cron: '30 2 * * *'
@@ -85,6 +89,8 @@ jobs:
       - name: Run picofuzz full chain
         run: |
           npm exec tsx --test tests/picofuzz/full_chain.test.ts
+        env:
+          STATE_BACKEND: ${{ github.event.inputs.backend || '' }}
 
       - name: Upload CSV results
         uses: actions/upload-artifact@v7

.github/workflows/picofuzz.yml

@@ -11,6 +11,10 @@ on:
         description: 'typeberry version: `next` (default), `<semver>-<sha>` (pre-release), or `<semver>` (release)'
         required: false
         default: 'next'
+      backend:
+        description: 'State backend: fjall, lmdb, fjall-hybrid, lmdb-hybrid (default: version default)'
+        required: false
+        default: ''
   repository_dispatch:
   schedule:
     # 2x/day (reduced from 5x) — staggered overnight to relieve the single shared self-hosted runner.
@@ -100,6 +104,8 @@ jobs:
       - name: Run picofuzz ${{ matrix.test }}
         run: |
           npm exec tsx --test tests/picofuzz/${{ matrix.test }}.test.ts
+        env:
+          STATE_BACKEND: ${{ github.event.inputs.backend || '' }}
 
       - name: Upload CSV results
         uses: actions/upload-artifact@v7

tests/common.ts

@@ -60,6 +60,8 @@ for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
  */
 export const TYPEBERRY_IMAGE = process.env.TYPEBERRY_IMAGE ?? "typeberry:test";
 
+const STATE_BACKEND = process.env.STATE_BACKEND ?? "";
+
 const DOCKER_OPTIONS = (mem = "512m") => [
   "--cpu-shares",
   "2048",
@@ -146,6 +148,9 @@ export async function typeberry({
 }) {
   const containerName = uniqueContainerName("typeberry");
   trackedContainers.add(containerName);
+  if (STATE_BACKEND && !options.inMemory) {
+    dockerArgs = [...dockerArgs, "-e", `JAM_FUZZ_DB=${STATE_BACKEND}`];
+  }
   // Global config directives applied before the `fuzz-target` subcommand, in
   // order. `--config=default` is the implicit default; we list it explicitly so
   // jq-style overrides have a base config to layer onto.
@@ -157,13 +162,6 @@ export async function typeberry({
     configArgs.push('--config=.flavor="full"');
   }
   if (options.inMemory === true) {
-    // Force a pure in-memory state db. The default config sets
-    // `database_base_path: "./database"`, which the fuzz-target inherits and
-    // turns into an on-disk LMDB/fjall store; clearing it makes the node's
-    // resolveFuzzDbBase() return undefined → in-memory backend. Used by the
-    // conformance suite, whose short per-vector genesis resets are slow on the
-    // on-disk fuzz db and don't need its heap-bounding (state roots are
-    // storage-agnostic, so correctness is unaffected).
     configArgs.push('--config=.database_base_path="undefined"');
   }
   const typeberry = ExternalProcess.spawn(

tests/docker-imports-full.test.ts

@@ -8,6 +8,7 @@ import { ExternalProcess } from "./external-process.js";
 // so this is sized well above that. Kept under the workflow's 390-min job cap so
 // the test self-aborts and reaps its container before GitHub hard-kills the job.
 const TEST_TIMEOUT = 360 * 60 * 1_000;
+const STATE_BACKEND = process.env.STATE_BACKEND ?? "";
 
 // The dump holds blocks for time slots 0..100051; typeberry logs the slot of
 // the best block, so reaching the last block prints `Best: #100051`
@@ -33,6 +34,7 @@ describe("Docker image can import the full-chainspec block dump", { timeout: TES
       "--config=default",
       '--config=.flavor="full"',
       "--config=.chain_spec=/block-dumps/full/chain-spec-full.json",
+      ...(STATE_BACKEND ? [`--config=.state_backend="${STATE_BACKEND}"`] : []),
       "import",
       "/block-dumps/full/chain-100k.bin",
     )

tests/docker-imports.test.ts

@@ -3,6 +3,7 @@ import { TYPEBERRY_IMAGE } from "./common.js";
 import { ExternalProcess } from "./external-process.js";
 
 const TEST_TIMEOUT = 5 * 60 * 1_000;
+const STATE_BACKEND = process.env.STATE_BACKEND ?? "";
 
 describe("Docker image can import block dumps", { timeout: TEST_TIMEOUT }, () => {
   ["fallback", "safrole", "storage", "storage_light"].forEach((dir) => {
@@ -17,6 +18,7 @@ describe("Docker image can import block dumps", { timeout: TEST_TIMEOUT }, () =>
         TYPEBERRY_IMAGE,
         "--config=default",
         "--config=.chain_spec=/block-dumps/chain-spec.json",
+        ...(STATE_BACKEND ? [`--config=.state_backend="${STATE_BACKEND}"`] : []),
         "import",
         `/block-dumps/${dir}.bin`,
       ).terminateAfter(TEST_TIMEOUT);

tests/picofuzz/conformance.test.ts

@@ -9,10 +9,6 @@ runPicofuzzTest("conformance", EXAMPLES_DIR, {
   noLogs: true,
   ignore: [],
   highMemory: true,
-  // Run with a pure in-memory state db. Conformance re-initializes genesis state
-  // on every vector, which is pathologically slow on the on-disk fuzz db (the
-  // genesis reset re-opens/wipes the keyspace each time) — the on-disk fjall
-  // backend pushed this suite past its timeout. State roots don't depend on the
-  // storage backend, so in-memory keeps conformance correct and fast.
   inMemory: true,
+  timeoutMs: 45 * 60 * 1000,
 });