From f12e76a13a7b181c789a9c50d17f200b193da72c Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Fri, 20 Mar 2026 13:09:10 +0100 Subject: [PATCH 01/58] feat(benchmark): target real note tree sync paths --- README.md | 3 + docs/BENCHMARKS.md | 181 ++++ package.json | 14 +- packages/treecrdt-benchmark/src/sync.ts | 593 ++++++++++-- packages/treecrdt-sqlite-node/package.json | 12 +- .../scripts/bench-note-paths.ts | 446 +++++++++ .../scripts/bench-sync.ts | 856 +++++++++++++++--- pnpm-lock.yaml | 15 + scripts/run-sync-bench.mjs | 181 ++++ 9 files changed, 2091 insertions(+), 210 deletions(-) create mode 100644 docs/BENCHMARKS.md create mode 100644 packages/treecrdt-sqlite-node/scripts/bench-note-paths.ts create mode 100644 scripts/run-sync-bench.mjs diff --git a/README.md b/README.md index 37067e27..b6c1b613 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,9 @@ pnpm build pnpm test ``` +## Benchmarks +For benchmark commands, product-facing note/sync scenarios, and the sync target matrix (`direct`, local Postgres sync server, remote sync server), see [docs/BENCHMARKS.md](docs/BENCHMARKS.md). + ## Playground - Live demo (GitHub Pages): https://cybersemics.github.io/treecrdt/ diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 00000000..b422220b --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,181 @@ +# Benchmarks + +The benchmark suite is most useful when you treat it as a set of product questions, not just a set of packages. + +## Start Here + +From the repo root: + +```sh +pnpm benchmark +pnpm benchmark:sync:help +``` + +Useful top-level entrypoints: + +```sh +pnpm benchmark +pnpm benchmark:sqlite-node +pnpm benchmark:sqlite-node:ops +pnpm benchmark:sqlite-node:note-paths +pnpm benchmark:sync +pnpm benchmark:sync:direct +pnpm benchmark:sync:local +pnpm benchmark:sync:remote +pnpm benchmark:web +pnpm benchmark:wasm +pnpm benchmark:postgres +``` + +`pnpm benchmark` writes JSON results under `benchmarks/`. + +## Which Benchmark Answers What? + +- First view on a new device, structure only: `benchmark:sync:*` with `sync-balanced-children-cold-start` +- First view on a new device, with payloads: `benchmark:sync:*` with `sync-balanced-children-payloads-cold-start` +- Local render cost after the data is already present: `benchmark:sqlite-node:note-paths -- --benches=read-children-payloads` +- Local mutation cost inside a large existing tree: `benchmark:sqlite-node:note-paths -- --benches=insert-into-large-tree` +- Protocol/storage baselines and worst-case stress: `sync-one-missing`, `sync-all`, `sync-children*`, `sync-root-children-fanout10` + +That split is intentional: + +- The sync benches answer "how long until the needed subtree data is in the local store?" +- The note-path benches answer "once the data is local, how quickly can the app render and mutate it?" + +## Recommended Product-Facing Runs + +### First View Sync + +Balanced-tree cold-start sync is the closest current benchmark to "open a node on a fresh device and load the first visible page". + +```sh +pnpm benchmark:sync:direct -- \ + --workloads=sync-balanced-children-cold-start,sync-balanced-children-payloads-cold-start \ + --counts=10000,50000,100000 \ + --fanout=10 +``` + +```sh +pnpm sync-server:postgres:db:start +pnpm benchmark:sync:local -- \ + --workloads=sync-balanced-children-cold-start,sync-balanced-children-payloads-cold-start \ + --counts=10000,50000,100000 \ + --fanout=10 +pnpm sync-server:postgres:db:stop +``` + +```sh +TREECRDT_SYNC_SERVER_URL=ws://host-or-elb/sync \ +pnpm benchmark:sync:remote -- \ + --workloads=sync-balanced-children-cold-start,sync-balanced-children-payloads-cold-start \ + --counts=10000,50000,100000 \ + --fanout=10 +``` + +Use `--fanout=20` when you want to model a broader notebook tree. + +### Local First View Read Path + +This measures the app-shaped local read immediately after sync: fetch the visible children page plus payloads for the parent and those children. + +```sh +pnpm benchmark:sqlite-node:note-paths -- \ + --benches=read-children-payloads \ + --counts=10000,50000,100000 \ + --fanout=10 \ + --page-size=10 \ + --payload-bytes=512 +``` + +### Local Mutation in a Large Tree + +This measures inserting one node with a payload into an already-large balanced tree. + +```sh +pnpm benchmark:sqlite-node:note-paths -- \ + --benches=insert-into-large-tree \ + --counts=10000,50000,100000 \ + --fanout=10 \ + --payload-bytes=512 +``` + +## Sync Targets + +The sync runner supports the same workload definitions across multiple environments: + +- `direct`: in-memory connected peers, no sync server +- `local-postgres-sync-server`: local WebSocket sync server backed by Postgres +- `remote-sync-server`: remote WebSocket sync server + +That keeps the workload constant while you compare transport and backend behavior. + +### Local Postgres Defaults + +```sh +postgres://postgres:postgres@127.0.0.1:5432/postgres +``` + +Override with `TREECRDT_POSTGRES_URL` or `--postgres-url=...`. + +### Remote Sync Server URL + +The remote URL is intentionally not hardcoded in this repo. Different deployments can have different latency, auth, retention, and scaling settings, so pass it at runtime through `TREECRDT_SYNC_SERVER_URL` or `--sync-server-url=...`. + +For public HTTPS deployments, prefer `wss://.../sync`. Use `ws://.../sync` for local or other plain HTTP deployments. + +## Current Sync Workloads + +Current sync workload definitions live in `packages/treecrdt-benchmark/src/sync.ts`. + +Product-facing defaults: + +- `sync-one-missing`: narrow protocol baseline for a tiny delta +- `sync-balanced-children-cold-start`: new device already knows the scope root and pulls the immediate children of a node from a balanced tree +- `sync-balanced-children-payloads-cold-start`: same balanced-tree cold-start path, plus payloads + +Specialized or synthetic workloads: + +- `sync-all`: overlapping divergent peers reconcile all ops +- `sync-children`: scoped sync against a synthetic high-fanout parent +- `sync-children-cold-start`: same synthetic high-fanout shape in one-way mode +- `sync-children-payloads`: synthetic high-fanout subtree with payloads +- `sync-children-payloads-cold-start`: one-way version of that same synthetic high-fanout payload case +- `sync-root-children-fanout10`: balanced-tree root-children delta with a move boundary case + +The `sync-children*` workloads are still worth keeping because they act as worst-case or stress-style scoped sync scenarios. They are not the best default proxy for normal note-taking, because they put a very large number of direct children under one parent. + +## Useful Flags + +All sync entrypoints forward arguments to `packages/treecrdt-sqlite-node/scripts/bench-sync.ts`. + +Common sync flags: + +- `--workloads=sync-balanced-children-cold-start,sync-balanced-children-payloads-cold-start` +- `--counts=100,1000,10000` +- `--count=1000` +- `--storages=memory,file` +- `--targets=direct,local-postgres-sync-server` +- `--fanout=10` +- `--sync-server-url=ws://host/sync` +- `--postgres-url=postgres://...` + +Common note-path flags: + +- `--benches=read-children-payloads,insert-into-large-tree` +- `--counts=10000,50000,100000` +- `--fanout=10` +- `--page-size=10` +- `--payload-bytes=512` + +## What Is Still Missing? + +The suite is much closer to real note-taking behavior now, but there is still one gap worth closing later: + +- a single end-to-end benchmark that measures cold-start sync and the first local render in one number + +Right now that path is covered in two pieces: + +- sync time via `sync-balanced-children*-cold-start` +- local read time via `read-children-payloads` + +That is already enough to identify where time is going, but an integrated "time to first visible page" benchmark would still be useful as a final product metric. diff --git a/package.json b/package.json index e01cbe3b..1bf3b314 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,21 @@ "build": "pnpm -r --if-present run build", "test:browser": "pnpm -C packages/treecrdt-wa-sqlite/e2e test:e2e", "test:native-node": "pnpm -C packages/sync/material/sqlite run test && pnpm -C packages/treecrdt-sqlite-node run test", - "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm -C packages/treecrdt-benchmark run build && pnpm -C packages/treecrdt-wa-sqlite/e2e run bench && pnpm -C packages/treecrdt-sqlite-node run benchmark && pnpm -C packages/treecrdt-wasm-js run benchmark && (if [ -n \"$TREECRDT_POSTGRES_URL\" ]; then pnpm -C packages/treecrdt-postgres-napi run benchmark; else echo \"Skipping postgres-napi benchmark (TREECRDT_POSTGRES_URL not set)\"; fi) && pnpm run benchmark:core && pnpm run benchmark:aggregate", + "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm run benchmark:web && pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sync:direct && pnpm run benchmark:wasm && pnpm run benchmark:postgres && pnpm run benchmark:core && pnpm run benchmark:aggregate", + "benchmark:sqlite-node": "pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sqlite-node:sync", "benchmark:core": "cargo bench -p treecrdt-core --bench core --features bench", "benchmark:aggregate": "node scripts/aggregate-bench.mjs", + "benchmark:postgres": "(if [ -n \"$TREECRDT_POSTGRES_URL\" ]; then pnpm -C packages/treecrdt-postgres-napi run benchmark; else echo \"Skipping postgres-napi benchmark (TREECRDT_POSTGRES_URL not set)\"; fi)", + "benchmark:sqlite-node:note-paths": "pnpm -C packages/treecrdt-sqlite-node run benchmark:note-paths", + "benchmark:sqlite-node:ops": "pnpm -C packages/treecrdt-sqlite-node run benchmark:ops", + "benchmark:sqlite-node:sync": "pnpm run benchmark:sync:direct", + "benchmark:sync": "node scripts/run-sync-bench.mjs", + "benchmark:sync:help": "node scripts/run-sync-bench.mjs --help", + "benchmark:sync:direct": "node scripts/run-sync-bench.mjs direct", + "benchmark:sync:local": "node scripts/run-sync-bench.mjs local", + "benchmark:sync:remote": "node scripts/run-sync-bench.mjs remote", + "benchmark:wasm": "pnpm -C packages/treecrdt-wasm-js run benchmark", + "benchmark:web": "pnpm -C packages/treecrdt-wa-sqlite/e2e run bench", "test": "pnpm run test:browser && pnpm run test:native-node", "playground": "pnpm --filter @treecrdt/playground dev", "sync-server:postgres:setup": "pnpm --filter @treecrdt/postgres-napi... --filter @treecrdt/sync-server-postgres-node... run build", diff --git a/packages/treecrdt-benchmark/src/sync.ts b/packages/treecrdt-benchmark/src/sync.ts index 631b9fb8..01796dee 100644 --- a/packages/treecrdt-benchmark/src/sync.ts +++ b/packages/treecrdt-benchmark/src/sync.ts @@ -1,24 +1,43 @@ -import type { Operation, OperationKind, ReplicaId } from '@treecrdt/interface'; -import { nodeIdToBytes16 } from '@treecrdt/interface/ids'; -import { envIntList } from './stats.js'; -import { benchTiming } from './timing.js'; +import type { Operation, OperationKind, ReplicaId } from "@treecrdt/interface"; +import { nodeIdToBytes16 } from "@treecrdt/interface/ids"; +import { envIntList } from "./stats.js"; +import { benchTiming } from "./timing.js"; export type SyncBenchWorkload = - | 'sync-all' - | 'sync-children' - | 'sync-root-children-fanout10' - | 'sync-one-missing'; + | "sync-all" + | "sync-balanced-children-cold-start" + | "sync-balanced-children-payloads-cold-start" + | "sync-children" + | "sync-children-cold-start" + | "sync-children-payloads" + | "sync-children-payloads-cold-start" + | "sync-root-children-fanout10" + | "sync-one-missing"; export const DEFAULT_SYNC_BENCH_SIZES = [100, 1000, 10_000] as const; export const DEFAULT_SYNC_BENCH_ROOT_CHILDREN_SIZES = [1110] as const; +export const DEFAULT_SYNC_BENCH_FANOUT = 10; +export const DEFAULT_SYNC_BENCH_PAGE_SIZE = 10; +export const DEFAULT_SYNC_BENCH_PAYLOAD_BYTES = 512; export const DEFAULT_SYNC_BENCH_WORKLOADS = [ - 'sync-all', - 'sync-children', - 'sync-one-missing', + "sync-one-missing", + "sync-balanced-children-cold-start", + "sync-balanced-children-payloads-cold-start", +] as const satisfies readonly SyncBenchWorkload[]; +export const SYNTHETIC_SYNC_BENCH_WORKLOADS = [ + "sync-all", + "sync-children", + "sync-children-cold-start", + "sync-children-payloads", + "sync-children-payloads-cold-start", +] as const satisfies readonly SyncBenchWorkload[]; +export const ALL_SYNC_BENCH_WORKLOADS = [ + ...DEFAULT_SYNC_BENCH_WORKLOADS, + ...SYNTHETIC_SYNC_BENCH_WORKLOADS, ] as const satisfies readonly SyncBenchWorkload[]; export const DEFAULT_SYNC_BENCH_ROOT_CHILDREN_WORKLOADS = [ - 'sync-root-children-fanout10', + "sync-root-children-fanout10", ] as const satisfies readonly SyncBenchWorkload[]; export const SYNC_BENCH_DEFAULT_MAX_CODEWORDS = 200_000; @@ -26,28 +45,24 @@ export const SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE = 2048; export const SYNC_BENCH_DEFAULT_SUBSCRIBE_CODEWORDS_PER_MESSAGE = 1024; export function syncBenchSizesFromEnv(): number[] { - return envIntList('SYNC_BENCH_SIZES') ?? Array.from(DEFAULT_SYNC_BENCH_SIZES); + return envIntList("SYNC_BENCH_SIZES") ?? Array.from(DEFAULT_SYNC_BENCH_SIZES); } export function syncBenchRootChildrenSizesFromEnv(): number[] { - return ( - envIntList('SYNC_BENCH_ROOT_CHILDREN_SIZES') ?? - Array.from(DEFAULT_SYNC_BENCH_ROOT_CHILDREN_SIZES) - ); + return envIntList("SYNC_BENCH_ROOT_CHILDREN_SIZES") ?? Array.from(DEFAULT_SYNC_BENCH_ROOT_CHILDREN_SIZES); } -export function syncBenchTiming(opts: { defaultIterations?: number } = {}): { - iterations: number; - warmupIterations: number; -} { +export function syncBenchTiming(opts: { defaultIterations?: number } = {}): { iterations: number; warmupIterations: number } { return benchTiming({ - iterationsEnv: ['SYNC_BENCH_ITERATIONS', 'BENCH_ITERATIONS'], - warmupEnv: ['SYNC_BENCH_WARMUP', 'BENCH_WARMUP'], + iterationsEnv: ["SYNC_BENCH_ITERATIONS", "BENCH_ITERATIONS"], + warmupEnv: ["SYNC_BENCH_WARMUP", "BENCH_WARMUP"], defaultIterations: opts.defaultIterations ?? 10, }); } -export type SyncFilter = { all: Record } | { children: { parent: Uint8Array } }; +export type SyncFilter = + | { all: Record } + | { children: { parent: Uint8Array } }; export type SyncBenchCase = { name: string; @@ -62,7 +77,7 @@ export type SyncBenchCase = { export function nodeIdFromInt(i: number): string { if (!Number.isInteger(i) || i < 0) throw new Error(`invalid node id: ${i}`); - return i.toString(16).padStart(32, '0'); + return i.toString(16).padStart(32, "0"); } function orderKeyFromPosition(position: number): Uint8Array { @@ -76,7 +91,7 @@ function orderKeyFromPosition(position: number): Uint8Array { function replicaFromLabel(label: string): Uint8Array { const encoded = new TextEncoder().encode(label); - if (encoded.length === 0) throw new Error('label must not be empty'); + if (encoded.length === 0) throw new Error("label must not be empty"); const out = new Uint8Array(32); for (let i = 0; i < out.length; i += 1) out[i] = encoded[i % encoded.length]!; return out; @@ -86,7 +101,7 @@ export function makeOp( replica: ReplicaId, counter: number, lamport: number, - kind: OperationKind, + kind: OperationKind ): Operation { return { meta: { id: { replica, counter }, lamport }, kind }; } @@ -97,6 +112,16 @@ export function maxLamport(ops: Operation[]): number { type ParentCursor = { parent: string; nextChildPosition: number }; +function payloadBytesFromSeed(seed: number, size = 512): Uint8Array { + if (!Number.isInteger(seed) || seed < 0) throw new Error(`invalid payload seed: ${seed}`); + if (!Number.isInteger(size) || size <= 0) throw new Error(`invalid payload size: ${size}`); + const out = new Uint8Array(size); + for (let i = 0; i < out.length; i += 1) { + out[i] = (seed + i * 31) % 251; + } + return out; +} + export function buildFanoutInsertTreeOps(opts: { replica: ReplicaId; size: number; @@ -104,15 +129,14 @@ export function buildFanoutInsertTreeOps(opts: { root: string; }): Operation[] { if (!Number.isInteger(opts.size) || opts.size <= 0) throw new Error(`invalid size: ${opts.size}`); - if (!Number.isInteger(opts.fanout) || opts.fanout <= 0) - throw new Error(`invalid fanout: ${opts.fanout}`); + if (!Number.isInteger(opts.fanout) || opts.fanout <= 0) throw new Error(`invalid fanout: ${opts.fanout}`); const ops: Operation[] = []; const queue: ParentCursor[] = [{ parent: opts.root, nextChildPosition: 0 }]; for (let i = 1; i <= opts.size; i += 1) { const cursor = queue[0]; - if (!cursor) throw new Error('fanout tree queue empty'); + if (!cursor) throw new Error("fanout tree queue empty"); const parent = cursor.parent; const position = cursor.nextChildPosition; @@ -121,12 +145,7 @@ export function buildFanoutInsertTreeOps(opts: { const node = nodeIdFromInt(i); ops.push( - makeOp(opts.replica, i, i, { - type: 'insert', - parent, - node, - orderKey: orderKeyFromPosition(position), - }), + makeOp(opts.replica, i, i, { type: "insert", parent, node, orderKey: orderKeyFromPosition(position) }) ); queue.push({ parent: node, nextChildPosition: 0 }); } @@ -134,27 +153,105 @@ export function buildFanoutInsertTreeOps(opts: { return ops; } +function targetChildrenForFirstChild(treeSize: number, fanout: number): string[] { + const childCount = Math.min(fanout, Math.max(0, treeSize - fanout)); + return Array.from({ length: childCount }, (_, i) => nodeIdFromInt(fanout + i + 1)); +} + +function buildBalancedChildrenColdStartCase(opts: { + size: number; + fanout: number; + replicas: { s: ReplicaId; p: ReplicaId }; + root: string; + payloadBytes: number; + withPayloads: boolean; +}): SyncBenchCase { + const treeSize = opts.size; + if (!Number.isInteger(treeSize) || treeSize <= opts.fanout) { + throw new Error(`balanced children cold-start requires size > fanout (${opts.fanout})`); + } + + const sharedOps = buildFanoutInsertTreeOps({ + replica: opts.replicas.s, + size: treeSize, + fanout: opts.fanout, + root: opts.root, + }); + const scopeRootInsert = sharedOps[0]; + if (!scopeRootInsert || scopeRootInsert.kind.type !== "insert") { + throw new Error("expected balanced tree seed to start with scope root insert"); + } + + const targetParent = scopeRootInsert.kind.node; + const targetChildren = targetChildrenForFirstChild(treeSize, opts.fanout); + const opsA: Operation[] = [scopeRootInsert]; + const opsB: Operation[] = [...sharedOps]; + + if (opts.withPayloads) { + let counter = 0; + let lamport = maxLamport(sharedOps); + for (let i = 0; i < sharedOps.length; i += 1) { + const op = sharedOps[i]; + if (op?.kind.type !== "insert") continue; + opsB.push( + makeOp(opts.replicas.p, ++counter, ++lamport, { + type: "payload", + node: op.kind.node, + payload: payloadBytesFromSeed(i + 1, opts.payloadBytes), + }) + ); + } + } + + const transferredOps = opts.withPayloads ? 1 + targetChildren.length * 2 : targetChildren.length; + return { + name: `sync-balanced-children${opts.withPayloads ? "-payloads" : ""}-cold-start-fanout${opts.fanout}-${treeSize}`, + opsA, + opsB, + filter: { children: { parent: nodeIdToBytes16(targetParent) } }, + totalOps: transferredOps, + extra: { + treeSize, + fanout: opts.fanout, + targetParent, + targetDepth: 1, + targetChildren: targetChildren.length, + coldStart: true, + balancedTree: true, + knownScopeRoot: true, + payloadBytes: opts.withPayloads ? opts.payloadBytes : 0, + payloadsEverywhere: opts.withPayloads, + pageSize: Math.min(DEFAULT_SYNC_BENCH_PAGE_SIZE, targetChildren.length), + }, + expectedFinalOpsA: opsA.length + transferredOps, + expectedFinalOpsB: opsB.length, + }; +} + export function buildSyncBenchCase(opts: { workload: SyncBenchWorkload; size: number; fanout?: number; + payloadBytes?: number; }): SyncBenchCase { const { workload } = opts; const size = opts.size; - const root = '0'.repeat(32); + const root = "0".repeat(32); const replicas = { - a: replicaFromLabel('a'), - b: replicaFromLabel('b'), - m: replicaFromLabel('m'), - s: replicaFromLabel('s'), - x: replicaFromLabel('x'), - y: replicaFromLabel('y'), + a: replicaFromLabel("a"), + b: replicaFromLabel("b"), + m: replicaFromLabel("m"), + p: replicaFromLabel("p"), + s: replicaFromLabel("s"), + x: replicaFromLabel("x"), + y: replicaFromLabel("y"), }; + const fanout = opts.fanout ?? DEFAULT_SYNC_BENCH_FANOUT; + const payloadBytes = opts.payloadBytes ?? DEFAULT_SYNC_BENCH_PAYLOAD_BYTES; - if (workload === 'sync-one-missing') { + if (workload === "sync-one-missing") { const treeSize = size; - if (!Number.isInteger(treeSize) || treeSize <= 0) - throw new Error(`sync-one-missing requires size > 0`); + if (!Number.isInteger(treeSize) || treeSize <= 0) throw new Error(`sync-one-missing requires size > 0`); const missingCounter = Math.min(treeSize, Math.max(1, Math.ceil(treeSize / 2))); const missingNode = nodeIdFromInt(missingCounter); @@ -163,7 +260,7 @@ export function buildSyncBenchCase(opts: { const opsA: Operation[] = []; for (let counter = 1; counter <= treeSize; counter += 1) { const op = makeOp(replicas.s, counter, counter, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(counter), orderKey: orderKeyFromPosition(counter - 1), @@ -184,28 +281,43 @@ export function buildSyncBenchCase(opts: { }; } - if (workload === 'sync-root-children-fanout10') { - const fanout = opts.fanout ?? 10; - const treeSize = size; - if (treeSize < fanout) - throw new Error(`sync-root-children-fanout10 requires size >= ${fanout}`); + if (workload === "sync-balanced-children-cold-start") { + return buildBalancedChildrenColdStartCase({ + size, + fanout, + replicas: { s: replicas.s, p: replicas.p }, + root, + payloadBytes, + withPayloads: false, + }); + } - const sharedOps = buildFanoutInsertTreeOps({ - replica: replicas.s, - size: treeSize, + if (workload === "sync-balanced-children-payloads-cold-start") { + return buildBalancedChildrenColdStartCase({ + size, fanout, + replicas: { s: replicas.s, p: replicas.p }, root, + payloadBytes, + withPayloads: true, }); + } + + if (workload === "sync-root-children-fanout10") { + const treeSize = size; + if (treeSize < fanout) throw new Error(`sync-root-children-fanout10 requires size >= ${fanout}`); + + const sharedOps = buildFanoutInsertTreeOps({ replica: replicas.s, size: treeSize, fanout, root }); const movedNode = nodeIdFromInt(fanout); - const trash = 'f'.repeat(32); + const trash = "f".repeat(32); const moveOut = makeOp(replicas.m, 1, treeSize + 1, { - type: 'move', + type: "move", node: movedNode, newParent: trash, orderKey: orderKeyFromPosition(0), }); const moveBack = makeOp(replicas.m, 2, treeSize + 2, { - type: 'move', + type: "move", node: movedNode, newParent: root, orderKey: orderKeyFromPosition(fanout - 1), @@ -215,7 +327,7 @@ export function buildSyncBenchCase(opts: { const opsB = sharedOps; const totalOps = 2; return { - name: `sync-root-children-fanout10-${treeSize}`, + name: `sync-root-children-fanout${fanout}-${treeSize}`, opsA, opsB, filter: { children: { parent: nodeIdToBytes16(root) } }, @@ -226,7 +338,7 @@ export function buildSyncBenchCase(opts: { }; } - if (workload === 'sync-all') { + if (workload === "sync-all") { const shared = Math.floor(size / 2); const unique = size - shared; const sharedOps: Operation[] = []; @@ -237,11 +349,11 @@ export function buildSyncBenchCase(opts: { const counter = i + 1; sharedOps.push( makeOp(replicas.s, counter, counter, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(counter), orderKey: orderKeyFromPosition(i), - }), + }) ); } for (let i = 0; i < unique; i++) { @@ -249,19 +361,19 @@ export function buildSyncBenchCase(opts: { const lamport = shared + counter; aOps.push( makeOp(replicas.a, counter, lamport, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(shared + counter), orderKey: orderKeyFromPosition(shared + i), - }), + }) ); bOps.push( makeOp(replicas.b, counter, lamport, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(shared + unique + counter), orderKey: orderKeyFromPosition(shared + i), - }), + }) ); } @@ -281,9 +393,334 @@ export function buildSyncBenchCase(opts: { }; } + if (workload === "sync-children-payloads") { + const targetParentHex = "a0".repeat(16); + const otherParentHex = "b0".repeat(16); + const targetCount = Math.floor(size / 2); + const otherCount = size - targetCount; + const sharedTarget = Math.floor(targetCount / 2); + const uniqueTarget = targetCount - sharedTarget; + const sharedOps: Operation[] = []; + const aTarget: Operation[] = []; + const bTarget: Operation[] = []; + const aOther: Operation[] = []; + const bOther: Operation[] = []; + + let lamport = 0; + let counterS = 0; + let counterA = 0; + let counterB = 0; + let counterX = 0; + let counterY = 0; + + sharedOps.push( + makeOp(replicas.s, ++counterS, ++lamport, { + type: "insert", + parent: root, + node: targetParentHex, + orderKey: orderKeyFromPosition(0), + }) + ); + sharedOps.push( + makeOp(replicas.s, ++counterS, ++lamport, { + type: "insert", + parent: root, + node: otherParentHex, + orderKey: orderKeyFromPosition(1), + }) + ); + sharedOps.push( + makeOp(replicas.s, ++counterS, ++lamport, { + type: "payload", + node: targetParentHex, + payload: payloadBytesFromSeed(10_000, payloadBytes), + }) + ); + + bTarget.push( + makeOp(replicas.b, ++counterB, ++lamport, { + type: "payload", + node: targetParentHex, + payload: payloadBytesFromSeed(20_000, payloadBytes), + }) + ); + + for (let i = 0; i < sharedTarget; i += 1) { + const child = nodeIdFromInt(i + 1); + sharedOps.push( + makeOp(replicas.s, ++counterS, ++lamport, { + type: "insert", + parent: targetParentHex, + node: child, + orderKey: orderKeyFromPosition(i), + }) + ); + sharedOps.push( + makeOp(replicas.s, ++counterS, ++lamport, { + type: "payload", + node: child, + payload: payloadBytesFromSeed(i + 1, payloadBytes), + }) + ); + } + + for (let i = 0; i < uniqueTarget; i += 1) { + const position = sharedTarget + i; + const childA = nodeIdFromInt(sharedTarget + i + 1); + const childB = nodeIdFromInt(sharedTarget + uniqueTarget + i + 1); + + aTarget.push( + makeOp(replicas.a, ++counterA, ++lamport, { + type: "insert", + parent: targetParentHex, + node: childA, + orderKey: orderKeyFromPosition(position), + }) + ); + aTarget.push( + makeOp(replicas.a, ++counterA, ++lamport, { + type: "payload", + node: childA, + payload: payloadBytesFromSeed(30_000 + i, payloadBytes), + }) + ); + + bTarget.push( + makeOp(replicas.b, ++counterB, ++lamport, { + type: "insert", + parent: targetParentHex, + node: childB, + orderKey: orderKeyFromPosition(position), + }) + ); + bTarget.push( + makeOp(replicas.b, ++counterB, ++lamport, { + type: "payload", + node: childB, + payload: payloadBytesFromSeed(40_000 + i, payloadBytes), + }) + ); + } + + for (let i = 0; i < otherCount; i += 1) { + const childA = nodeIdFromInt(sharedTarget + 2 * uniqueTarget + i + 1); + const childB = nodeIdFromInt(sharedTarget + 2 * uniqueTarget + otherCount + i + 1); + + aOther.push( + makeOp(replicas.x, ++counterX, ++lamport, { + type: "insert", + parent: otherParentHex, + node: childA, + orderKey: orderKeyFromPosition(i), + }) + ); + aOther.push( + makeOp(replicas.x, ++counterX, ++lamport, { + type: "payload", + node: childA, + payload: payloadBytesFromSeed(50_000 + i, payloadBytes), + }) + ); + + bOther.push( + makeOp(replicas.y, ++counterY, ++lamport, { + type: "insert", + parent: otherParentHex, + node: childB, + orderKey: orderKeyFromPosition(i), + }) + ); + bOther.push( + makeOp(replicas.y, ++counterY, ++lamport, { + type: "payload", + node: childB, + payload: payloadBytesFromSeed(60_000 + i, payloadBytes), + }) + ); + } + + const opsA = [...sharedOps, ...aTarget, ...aOther]; + const opsB = [...sharedOps, ...bTarget, ...bOther]; + return { + name: `sync-children-payloads-${size}`, + opsA, + opsB, + filter: { children: { parent: nodeIdToBytes16(targetParentHex) } }, + totalOps: aTarget.length + bTarget.length, + extra: { + nodesPerPeer: size, + payloadBytes, + targetCount, + otherCount, + sharedTarget, + uniqueTarget, + parentPayloadRefresh: true, + }, + expectedFinalOpsA: opsA.length + bTarget.length, + expectedFinalOpsB: opsB.length + aTarget.length, + }; + } + + if (workload === "sync-children-payloads-cold-start") { + const targetParentHex = "a0".repeat(16); + const otherParentHex = "b0".repeat(16); + const targetCount = Math.floor(size / 2); + const otherCount = size - targetCount; + let lamport = 0; + let counterS = 0; + let counterB = 0; + let counterY = 0; + + const scopeRootInsert = makeOp(replicas.s, ++counterS, ++lamport, { + type: "insert", + parent: root, + node: targetParentHex, + orderKey: orderKeyFromPosition(0), + }); + const otherParentInsert = makeOp(replicas.s, ++counterS, ++lamport, { + type: "insert", + parent: root, + node: otherParentHex, + orderKey: orderKeyFromPosition(1), + }); + const scopeRootPayload = makeOp(replicas.b, ++counterB, ++lamport, { + type: "payload", + node: targetParentHex, + payload: payloadBytesFromSeed(20_000, payloadBytes), + }); + + const opsA: Operation[] = [scopeRootInsert]; + const opsB: Operation[] = [scopeRootInsert, otherParentInsert, scopeRootPayload]; + + for (let i = 0; i < targetCount; i += 1) { + const child = nodeIdFromInt(i + 1); + opsB.push( + makeOp(replicas.b, ++counterB, ++lamport, { + type: "insert", + parent: targetParentHex, + node: child, + orderKey: orderKeyFromPosition(i), + }) + ); + opsB.push( + makeOp(replicas.b, ++counterB, ++lamport, { + type: "payload", + node: child, + payload: payloadBytesFromSeed(30_000 + i, payloadBytes), + }) + ); + } + + for (let i = 0; i < otherCount; i += 1) { + const child = nodeIdFromInt(targetCount + i + 1); + opsB.push( + makeOp(replicas.y, ++counterY, ++lamport, { + type: "insert", + parent: otherParentHex, + node: child, + orderKey: orderKeyFromPosition(i), + }) + ); + opsB.push( + makeOp(replicas.y, ++counterY, ++lamport, { + type: "payload", + node: child, + payload: payloadBytesFromSeed(40_000 + i, payloadBytes), + }) + ); + } + + const transferredOps = 1 + targetCount * 2; + return { + name: `sync-children-payloads-cold-start-${size}`, + opsA, + opsB, + filter: { children: { parent: nodeIdToBytes16(targetParentHex) } }, + totalOps: transferredOps, + extra: { + nodesPerPeer: size, + payloadBytes, + targetCount, + otherCount, + coldStart: true, + knownScopeRoot: true, + parentPayloadRefresh: true, + }, + expectedFinalOpsA: opsA.length + transferredOps, + expectedFinalOpsB: opsB.length, + }; + } + + if (workload === "sync-children-cold-start") { + const targetParentHex = "a0".repeat(16); + const otherParentHex = "b0".repeat(16); + const targetCount = Math.floor(size / 2); + const otherCount = size - targetCount; + + let lamport = 0; + let counterS = 0; + let counterB = 0; + let counterY = 0; + + const scopeRootInsert = makeOp(replicas.s, ++counterS, ++lamport, { + type: "insert", + parent: root, + node: targetParentHex, + orderKey: orderKeyFromPosition(0), + }); + const otherParentInsert = makeOp(replicas.s, ++counterS, ++lamport, { + type: "insert", + parent: root, + node: otherParentHex, + orderKey: orderKeyFromPosition(1), + }); + + const opsA: Operation[] = [scopeRootInsert]; + const opsB: Operation[] = [scopeRootInsert, otherParentInsert]; + + for (let i = 0; i < targetCount; i += 1) { + opsB.push( + makeOp(replicas.b, ++counterB, ++lamport, { + type: "insert", + parent: targetParentHex, + node: nodeIdFromInt(i + 1), + orderKey: orderKeyFromPosition(i), + }) + ); + } + + for (let i = 0; i < otherCount; i += 1) { + opsB.push( + makeOp(replicas.y, ++counterY, ++lamport, { + type: "insert", + parent: otherParentHex, + node: nodeIdFromInt(targetCount + i + 1), + orderKey: orderKeyFromPosition(i), + }) + ); + } + + return { + name: `sync-children-cold-start-${size}`, + opsA, + opsB, + filter: { children: { parent: nodeIdToBytes16(targetParentHex) } }, + totalOps: targetCount, + extra: { + nodesPerPeer: size, + targetCount, + otherCount, + coldStart: true, + knownScopeRoot: true, + }, + expectedFinalOpsA: opsA.length + targetCount, + expectedFinalOpsB: opsB.length, + }; + } + // sync-children - const targetParentHex = 'a0'.repeat(16); - const otherParentHex = 'b0'.repeat(16); + const targetParentHex = "a0".repeat(16); + const otherParentHex = "b0".repeat(16); const targetCount = Math.floor(size / 2); const otherCount = size - targetCount; const sharedTarget = Math.floor(targetCount / 2); @@ -301,11 +738,11 @@ export function buildSyncBenchCase(opts: { const counter = i + 1; sharedOps.push( makeOp(replicas.s, counter, lamport, { - type: 'insert', + type: "insert", parent: targetParentHex, node: nodeIdFromInt(counter), orderKey: orderKeyFromPosition(i), - }), + }) ); } @@ -314,19 +751,19 @@ export function buildSyncBenchCase(opts: { const counter = i + 1; aTarget.push( makeOp(replicas.a, counter, lamport, { - type: 'insert', + type: "insert", parent: targetParentHex, node: nodeIdFromInt(sharedTarget + counter), orderKey: orderKeyFromPosition(sharedTarget + i), - }), + }) ); bTarget.push( makeOp(replicas.b, counter, lamport, { - type: 'insert', + type: "insert", parent: targetParentHex, node: nodeIdFromInt(sharedTarget + uniqueTarget + counter), orderKey: orderKeyFromPosition(sharedTarget + i), - }), + }) ); } @@ -334,19 +771,19 @@ export function buildSyncBenchCase(opts: { lamport += 1; aOther.push( makeOp(replicas.x, i + 1, lamport, { - type: 'insert', + type: "insert", parent: otherParentHex, node: nodeIdFromInt(sharedTarget + 2 * uniqueTarget + i + 1), orderKey: orderKeyFromPosition(i), - }), + }) ); bOther.push( makeOp(replicas.y, i + 1, lamport, { - type: 'insert', + type: "insert", parent: otherParentHex, node: nodeIdFromInt(sharedTarget + 2 * uniqueTarget + otherCount + i + 1), orderKey: orderKeyFromPosition(i), - }), + }) ); } diff --git a/packages/treecrdt-sqlite-node/package.json b/packages/treecrdt-sqlite-node/package.json index 2314da33..7dbac106 100644 --- a/packages/treecrdt-sqlite-node/package.json +++ b/packages/treecrdt-sqlite-node/package.json @@ -15,7 +15,10 @@ "build:ts": "tsc -p tsconfig.json", "build": "pnpm run build:native-ext && pnpm run copy:ext && pnpm run build:ts", "test": "pnpm -C ../sync/protocol run build && pnpm -C ../sync/material/sqlite run build && pnpm run build && vitest run", - "benchmark": "pnpm -C ../sync/protocol run build && pnpm -C ../sync/material/sqlite run build && pnpm run build && tsx ./scripts/bench.ts && tsx ./scripts/bench-sync.ts" + "benchmark:ops": "pnpm -C ../sync/protocol run build && pnpm -C ../sync/material/sqlite run build && pnpm run build && tsx ./scripts/bench.ts", + "benchmark:note-paths": "pnpm -C ../treecrdt-benchmark run build && pnpm run build && tsx ./scripts/bench-note-paths.ts", + "benchmark:sync": "pnpm -C ../treecrdt-benchmark run build && pnpm -C ../sync/protocol run build && pnpm -C ../sync/material/sqlite run build && pnpm -C ../treecrdt-postgres-napi run build && pnpm -C ../sync/server/postgres-node run build && pnpm run build && tsx ./scripts/bench-sync.ts", + "benchmark": "pnpm run benchmark:ops && pnpm run benchmark:note-paths && pnpm run benchmark:sync" }, "peerDependencies": { "better-sqlite3": "^9.0.0" @@ -30,12 +33,17 @@ "@types/node": "^20.19.25", "@treecrdt/auth": "workspace:*", "@treecrdt/benchmark": "workspace:*", + "@treecrdt/postgres-napi": "workspace:*", "@treecrdt/sqlite-conformance": "workspace:*", "@treecrdt/sync": "workspace:*", + "@treecrdt/sync-sqlite": "workspace:*", + "@treecrdt/sync-server-postgres-node": "workspace:*", "better-sqlite3": "^9.0.0", "cborg": "^4.3.2", + "@types/ws": "^8.18.1", "tsx": "^4.19.2", "typescript": "^5.9.3", - "vitest": "^1.6.0" + "vitest": "^1.6.0", + "ws": "^8.18.3" } } diff --git a/packages/treecrdt-sqlite-node/scripts/bench-note-paths.ts b/packages/treecrdt-sqlite-node/scripts/bench-note-paths.ts new file mode 100644 index 00000000..8fe4c690 --- /dev/null +++ b/packages/treecrdt-sqlite-node/scripts/bench-note-paths.ts @@ -0,0 +1,446 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +import Database from "better-sqlite3"; + +import { + buildFanoutInsertTreeOps, + nodeIdFromInt, + quantile, + runBenchmark, + type BenchmarkWorkload, +} from "@treecrdt/benchmark"; +import { repoRootFromImportMeta, writeResult } from "@treecrdt/benchmark/node"; +import type { Operation, ReplicaId } from "@treecrdt/interface"; +import { nodeIdToBytes16 } from "@treecrdt/interface/ids"; + +import { + createTreecrdtClient, + createSqliteNodeApi, + loadTreecrdtExtension, +} from "../dist/index.js"; + +type StorageKind = "memory" | "file"; +type NotePathBenchKind = "read-children-payloads" | "insert-into-large-tree"; +type ConfigEntry = [number, number]; + +const ALL_NOTE_PATH_BENCHES = [ + "read-children-payloads", + "insert-into-large-tree", +] as const satisfies readonly NotePathBenchKind[]; +const NOTE_PATH_BENCH_CONFIG: ReadonlyArray = [ + [10_000, 3], + [50_000, 2], +]; + +const STORAGES: readonly StorageKind[] = ["memory", "file"]; +const DEFAULT_FANOUT = 10; +const DEFAULT_PAGE_SIZE = 10; +const DEFAULT_PAYLOAD_BYTES = 512; +const ROOT = "0".repeat(32); + +function envInt(name: string): number | undefined { + const raw = process.env[name]; + if (raw == null || raw === "") return undefined; + const n = Number(raw); + return Number.isFinite(n) ? n : undefined; +} + +function parseConfigFromArgv(argv: string[]): Array | null { + let customConfig: Array | null = null; + const defaultIterations = Math.max(1, envInt("BENCH_ITERATIONS") ?? 1); + for (const arg of argv) { + if (arg.startsWith("--count=")) { + const val = arg.slice("--count=".length).trim(); + const count = val ? Number(val) : 10_000; + customConfig = [[Number.isFinite(count) && count > 0 ? count : 10_000, defaultIterations]]; + break; + } + if (arg.startsWith("--counts=")) { + const vals = arg + .slice("--counts=".length) + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + const parsed = vals + .map((s) => { + const n = Number(s); + return Number.isFinite(n) && n > 0 ? n : null; + }) + .filter((n): n is number => n != null) + .map((count) => [count, defaultIterations] as ConfigEntry); + if (parsed.length > 0) customConfig = parsed; + break; + } + } + return customConfig; +} + +function parseFlagValue(argv: string[], flag: string): string | undefined { + const prefix = `${flag}=`; + const raw = argv.find((arg) => arg.startsWith(prefix)); + return raw ? raw.slice(prefix.length).trim() : undefined; +} + +function parsePositiveIntFlag(argv: string[], flag: string, envName: string, fallback: number): number { + const raw = parseFlagValue(argv, flag) ?? process.env[envName]; + if (!raw) return fallback; + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`invalid ${flag} value "${raw}", expected a positive integer`); + } + return value; +} + +function parseKinds(argv: string[]): NotePathBenchKind[] { + const raw = + parseFlagValue(argv, "--benches") ?? + parseFlagValue(argv, "--bench") ?? + process.env.NOTE_PATH_BENCHES ?? + process.env.NOTE_PATH_BENCH; + if (!raw) return Array.from(ALL_NOTE_PATH_BENCHES); + + const seen = new Set(); + for (const value of raw.split(",").map((part) => part.trim()).filter((part) => part.length > 0)) { + if (!(ALL_NOTE_PATH_BENCHES as readonly string[]).includes(value)) { + throw new Error(`invalid note-path bench "${value}", expected one of: ${ALL_NOTE_PATH_BENCHES.join(", ")}`); + } + seen.add(value as NotePathBenchKind); + } + return seen.size > 0 ? Array.from(seen) : Array.from(ALL_NOTE_PATH_BENCHES); +} + +function payloadBytesFromSeed(seed: number, size = DEFAULT_PAYLOAD_BYTES): Uint8Array { + if (!Number.isInteger(seed) || seed < 0) throw new Error(`invalid payload seed: ${seed}`); + const out = new Uint8Array(size); + for (let i = 0; i < out.length; i += 1) { + out[i] = (seed + i * 31) % 251; + } + return out; +} + +function replicaFromLabel(label: string): Uint8Array { + const encoded = new TextEncoder().encode(label); + if (encoded.length === 0) throw new Error("label must not be empty"); + const out = new Uint8Array(32); + for (let i = 0; i < out.length; i += 1) out[i] = encoded[i % encoded.length]!; + return out; +} + +function makePayloadOp(replica: ReplicaId, counter: number, lamport: number, node: string, payload: Uint8Array): Operation { + return { + meta: { id: { replica, counter }, lamport }, + kind: { type: "payload", node, payload }, + }; +} + +function buildNotePathSeedOps(opts: { size: number; fanout: number; payloadBytes: number }) { + const replica = replicaFromLabel("bench"); + const insertOps = buildFanoutInsertTreeOps({ + replica, + size: opts.size, + fanout: opts.fanout, + root: ROOT, + }); + + const targetParent = nodeIdFromInt(1); + const targetChildrenCount = Math.min(opts.fanout, Math.max(0, opts.size - opts.fanout)); + const targetChildren = Array.from({ length: targetChildrenCount }, (_, i) => nodeIdFromInt(opts.fanout + i + 1)); + + const payloadOps: Operation[] = []; + let counter = insertOps.length; + let lamport = insertOps.length; + + payloadOps.push( + makePayloadOp(replica, ++counter, ++lamport, targetParent, payloadBytesFromSeed(10_000, opts.payloadBytes)) + ); + for (let i = 0; i < targetChildren.length; i += 1) { + payloadOps.push( + makePayloadOp( + replica, + ++counter, + ++lamport, + targetChildren[i]!, + payloadBytesFromSeed(20_000 + i, opts.payloadBytes) + ) + ); + } + + return { + ops: [...insertOps, ...payloadOps], + targetParent, + targetChildren, + }; +} + +async function openSeededClient(opts: { + repoRoot: string; + storage: StorageKind; + bench: NotePathBenchKind; + size: number; + seedOps: Operation[]; +}): Promise>> { + const dbPath = + opts.storage === "memory" + ? ":memory:" + : path.join( + opts.repoRoot, + "tmp", + "sqlite-node-note-paths", + `${opts.bench}-${opts.size}-${crypto.randomUUID()}.db` + ); + + if (opts.storage === "file") { + await fs.mkdir(path.dirname(dbPath), { recursive: true }); + } + + const db = new Database(dbPath); + loadTreecrdtExtension(db); + const api = createSqliteNodeApi(db); + await api.setDocId("treecrdt-note-paths-bench"); + await api.appendOps!( + opts.seedOps, + nodeIdToBytes16, + (replica) => replica + ); + + const client = await createTreecrdtClient(db, { docId: "treecrdt-note-paths-bench" }); + return { + ...client, + close: async () => { + await client.close(); + if (opts.storage === "file") { + await fs.rm(dbPath).catch(() => {}); + } + }, + }; +} + +function readChildrenPayloadsWorkload(opts: { + size: number; + targetParent: string; + expectedChildren: number; + pageSize: number; + payloadBytes: number; + fanout: number; +}): BenchmarkWorkload { + const visibleChildren = Math.min(opts.expectedChildren, opts.pageSize); + return { + name: `read-children-payloads-fanout${opts.fanout}-${opts.size}`, + totalOps: visibleChildren + 1, + iterations: 1, + warmupIterations: 0, + run: async (adapter: any) => { + const rows = await adapter.tree.childrenPage(opts.targetParent, null, opts.pageSize); + if (!Array.isArray(rows)) throw new Error("childrenPage did not return rows"); + if (rows.length !== visibleChildren) { + throw new Error(`expected ${visibleChildren} child rows, got ${rows.length}`); + } + + const parentPayload = await adapter.tree.getPayload(opts.targetParent); + if (!(parentPayload instanceof Uint8Array) || parentPayload.length !== opts.payloadBytes) { + throw new Error("target parent payload missing"); + } + + const payloads = await Promise.all(rows.map((row: { node: string }) => adapter.tree.getPayload(row.node))); + if (payloads.some((payload) => !(payload instanceof Uint8Array) || payload.length !== opts.payloadBytes)) { + throw new Error("one or more child payloads missing"); + } + + return { + extra: { + returnedChildren: rows.length, + targetChildren: opts.expectedChildren, + payloadBytes: opts.payloadBytes, + pageSize: opts.pageSize, + targetParent: opts.targetParent, + }, + }; + }, + }; +} + +function insertIntoLargeTreeWorkload(opts: { + size: number; + targetParent: string; + payloadBytes: number; + fanout: number; +}): BenchmarkWorkload { + const replica = replicaFromLabel("writer"); + const newNode = nodeIdFromInt(opts.size + 10_000); + const payload = payloadBytesFromSeed(90_000, opts.payloadBytes); + + return { + name: `insert-into-balanced-tree-fanout${opts.fanout}-${opts.size}`, + totalOps: 1, + iterations: 1, + warmupIterations: 0, + run: async (adapter: any) => { + const op = await adapter.local.insert(replica, opts.targetParent, newNode, { type: "last" }, payload); + if (op.kind.type !== "insert" || op.kind.node !== newNode) { + throw new Error("insert did not return the new node"); + } + + const parent = await adapter.tree.parent(newNode); + if (parent !== opts.targetParent) { + throw new Error(`inserted node parent mismatch: expected ${opts.targetParent}, got ${String(parent)}`); + } + + const storedPayload = await adapter.tree.getPayload(newNode); + if (!(storedPayload instanceof Uint8Array) || storedPayload.length !== opts.payloadBytes) { + throw new Error("inserted node payload missing"); + } + + return { + extra: { + payloadBytes: opts.payloadBytes, + targetParent: opts.targetParent, + insertedNode: newNode, + }, + }; + }, + }; +} + +async function runWorkload(opts: { + repoRoot: string; + storage: StorageKind; + bench: NotePathBenchKind; + size: number; + iterations: number; + fanout: number; + payloadBytes: number; + seedOps: Operation[]; + workload: BenchmarkWorkload; +}) { + let result: Awaited>; + const adapterFactory = async () => + (await openSeededClient({ + repoRoot: opts.repoRoot, + storage: opts.storage, + bench: opts.bench, + size: opts.size, + seedOps: opts.seedOps, + })) as any; + + if (opts.iterations > 1) { + const durations: number[] = []; + let lastExtra: Record | undefined; + for (let i = 0; i < opts.iterations; i += 1) { + const next = await runBenchmark(adapterFactory as any, opts.workload); + durations.push(next.durationMs); + lastExtra = next.extra; + } + const durationMs = quantile(durations, 0.5); + const totalOps = opts.workload.totalOps ?? -1; + result = { + name: opts.workload.name, + totalOps, + durationMs, + opsPerSec: + totalOps > 0 && durationMs > 0 + ? (totalOps / durationMs) * 1000 + : durationMs > 0 + ? 1000 / durationMs + : Infinity, + extra: { + ...(lastExtra ?? {}), + iterations: opts.iterations, + samplesMs: durations, + p95Ms: quantile(durations, 0.95), + minMs: Math.min(...durations), + maxMs: Math.max(...durations), + }, + }; + } else { + result = await runBenchmark(adapterFactory as any, opts.workload); + } + + const outFile = path.join( + opts.repoRoot, + "benchmarks", + "sqlite-node-note-paths", + `${opts.storage}-${result.name}.json` + ); + const payload = await writeResult(result, { + implementation: "sqlite-node", + storage: opts.storage, + workload: result.name, + outFile, + extra: { + count: opts.size, + bench: opts.bench, + fanout: opts.fanout, + payloadBytes: opts.payloadBytes, + ...result.extra, + }, + }); + console.log(JSON.stringify(payload, null, 2)); +} + +async function main() { + const repoRoot = repoRootFromImportMeta(import.meta.url, 3); + const argv = process.argv.slice(2); + const config = parseConfigFromArgv(argv) ?? [...NOTE_PATH_BENCH_CONFIG]; + const benches = parseKinds(argv); + const fanout = parsePositiveIntFlag(argv, "--fanout", "NOTE_PATH_BENCH_FANOUT", DEFAULT_FANOUT); + const pageSize = parsePositiveIntFlag(argv, "--page-size", "NOTE_PATH_BENCH_PAGE_SIZE", DEFAULT_PAGE_SIZE); + const payloadBytes = parsePositiveIntFlag( + argv, + "--payload-bytes", + "NOTE_PATH_BENCH_PAYLOAD_BYTES", + DEFAULT_PAYLOAD_BYTES + ); + + for (const [size, iterations] of config) { + const seed = buildNotePathSeedOps({ size, fanout, payloadBytes }); + const readWorkload = readChildrenPayloadsWorkload({ + size, + targetParent: seed.targetParent, + expectedChildren: seed.targetChildren.length, + pageSize, + payloadBytes, + fanout, + }); + const insertWorkload = insertIntoLargeTreeWorkload({ + size, + targetParent: seed.targetParent, + payloadBytes, + fanout, + }); + + for (const storage of STORAGES) { + if (benches.includes("read-children-payloads")) { + await runWorkload({ + repoRoot, + storage, + bench: "read-children-payloads", + size, + iterations, + fanout, + payloadBytes, + seedOps: seed.ops, + workload: readWorkload, + }); + } + if (benches.includes("insert-into-large-tree")) { + await runWorkload({ + repoRoot, + storage, + bench: "insert-into-large-tree", + size, + iterations, + fanout, + payloadBytes, + seedOps: seed.ops, + workload: insertWorkload, + }); + } + } + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index d7779244..c9fa3c27 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -1,8 +1,13 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; -import Database from 'better-sqlite3'; +import fs from "node:fs/promises"; +import net from "node:net"; +import path from "node:path"; +import Database from "better-sqlite3"; +import WebSocket from "ws"; + import { + ALL_SYNC_BENCH_WORKLOADS, buildSyncBenchCase, + DEFAULT_SYNC_BENCH_FANOUT, DEFAULT_SYNC_BENCH_ROOT_CHILDREN_WORKLOADS, DEFAULT_SYNC_BENCH_WORKLOADS, SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, @@ -10,22 +15,64 @@ import { maxLamport, quantile, type SyncBenchWorkload, -} from '@treecrdt/benchmark'; -import { repoRootFromImportMeta, writeResult } from '@treecrdt/benchmark/node'; -import type { Operation } from '@treecrdt/interface'; -import { decodeSqliteOpRefs, decodeSqliteOps } from '@treecrdt/interface/sqlite'; -import { nodeIdToBytes16 } from '@treecrdt/interface/ids'; +} from "@treecrdt/benchmark"; +import { repoRootFromImportMeta, writeResult } from "@treecrdt/benchmark/node"; +import type { Operation } from "@treecrdt/interface"; +import { nodeIdToBytes16 } from "@treecrdt/interface/ids"; +import { SyncPeer, type Filter } from "@treecrdt/sync"; import { createInMemoryConnectedPeers, makeQueuedSyncBackend, type FlushableSyncBackend, -} from '@treecrdt/sync/in-memory'; -import { treecrdtSyncV0ProtobufCodec } from '@treecrdt/sync/protobuf'; -import type { Filter } from '@treecrdt/sync'; -import { createSqliteNodeApi, loadTreecrdtExtension } from '../dist/index.js'; +} from "@treecrdt/sync/in-memory"; +import { createTreecrdtSyncBackendFromClient } from "@treecrdt/sync-sqlite/backend"; +import { treecrdtSyncV0ProtobufCodec } from "@treecrdt/sync/protobuf"; +import { + wrapDuplexTransportWithCodec, + type DuplexTransport, +} from "@treecrdt/sync/transport"; +import { startSyncServer } from "@treecrdt/sync-server-postgres-node"; + +import { + createTreecrdtClient, + createSqliteNodeApi, + loadTreecrdtExtension, +} from "../dist/index.js"; -type StorageKind = 'memory' | 'file'; +type StorageKind = "memory" | "file"; type ConfigEntry = [number, number]; +type SyncBenchTargetId = + | "direct" + | "local-postgres-sync-server" + | "remote-sync-server"; + +type BenchCase = { + storage: StorageKind; + target: SyncBenchTargetId; + workload: SyncBenchWorkload; + size: number; + iterations: number; + fanout: number; +}; + +type SyncBenchResult = { + name: string; + totalOps: number; + durationMs: number; + opsPerSec: number; + extra?: Record; +}; + +type SyncBenchConnection = { + transport: DuplexTransport; + close: () => Promise; +}; + +type SyncBenchTargetRuntime = { + id: Exclude; + connect: (docId: string) => Promise; + close: () => Promise; +}; const SYNC_BENCH_CONFIG: ReadonlyArray = [ [100, 10], @@ -35,27 +82,41 @@ const SYNC_BENCH_CONFIG: ReadonlyArray = [ const SYNC_BENCH_ROOT_CONFIG: ReadonlyArray = [[1110, 10]]; +const DEFAULT_TARGETS: readonly SyncBenchTargetId[] = ["direct"]; +const ALL_TARGETS: readonly SyncBenchTargetId[] = [ + "direct", + "local-postgres-sync-server", + "remote-sync-server", +]; +const DEFAULT_STORAGES: readonly StorageKind[] = ["memory", "file"]; +const ALL_WORKLOADS: readonly SyncBenchWorkload[] = [ + ...ALL_SYNC_BENCH_WORKLOADS, + ...DEFAULT_SYNC_BENCH_ROOT_CHILDREN_WORKLOADS, +]; +const SERVER_READY_TIMEOUT_MS = 10_000; +const SERVER_READY_POLL_MS = 100; + function envInt(name: string): number | undefined { const raw = process.env[name]; - if (raw == null || raw === '') return undefined; + if (raw == null || raw === "") return undefined; const n = Number(raw); return Number.isFinite(n) ? n : undefined; } function parseConfigFromArgv(argv: string[]): Array | null { let customConfig: Array | null = null; - const defaultIterations = Math.max(1, envInt('BENCH_ITERATIONS') ?? 1); + const defaultIterations = Math.max(1, envInt("BENCH_ITERATIONS") ?? 1); for (const arg of argv) { - if (arg.startsWith('--count=')) { - const val = arg.slice('--count='.length).trim(); + if (arg.startsWith("--count=")) { + const val = arg.slice("--count=".length).trim(); const count = val ? Number(val) : 500; customConfig = [[Number.isFinite(count) && count > 0 ? count : 500, defaultIterations]]; break; } - if (arg.startsWith('--counts=')) { + if (arg.startsWith("--counts=")) { const vals = arg - .slice('--counts='.length) - .split(',') + .slice("--counts=".length) + .split(",") .map((s) => s.trim()) .filter((s) => s.length > 0); const parsed = vals @@ -72,57 +133,245 @@ function parseConfigFromArgv(argv: string[]): Array | null { return customConfig; } -type BenchCase = { - storage: StorageKind; - workload: SyncBenchWorkload; - size: number; - iterations: number; -}; +function parseFlagValue(argv: string[], flag: string): string | undefined { + const prefix = `${flag}=`; + const raw = argv.find((arg) => arg.startsWith(prefix)); + return raw ? raw.slice(prefix.length).trim() : undefined; +} -type SyncBenchResult = { - name: string; - totalOps: number; - durationMs: number; - opsPerSec: number; - extra?: Record; -}; +function parsePositiveIntFlag(argv: string[], flag: string, envName: string, fallback: number): number { + const raw = parseFlagValue(argv, flag) ?? process.env[envName]; + if (!raw) return fallback; + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`invalid ${flag} value "${raw}", expected a positive integer`); + } + return value; +} + +function parseCsv(raw: string | undefined): string[] { + if (!raw) return []; + return raw + .split(",") + .map((value) => value.trim()) + .filter((value) => value.length > 0); +} + +function normalizeTargetId(raw: string): SyncBenchTargetId | null { + const value = raw.trim(); + if (value === "direct") return "direct"; + if ( + value === "local" || + value === "local-server" || + value === "local-postgres" || + value === "local-postgres-sync-server" + ) { + return "local-postgres-sync-server"; + } + if ( + value === "remote" || + value === "remote-server" || + value === "remote-sync" || + value === "remote-sync-server" + ) { + return "remote-sync-server"; + } + return null; +} + +function parseTargets(argv: string[]): SyncBenchTargetId[] { + const raw = + parseFlagValue(argv, "--targets") ?? + parseFlagValue(argv, "--target") ?? + process.env.SYNC_BENCH_TARGETS ?? + process.env.SYNC_BENCH_TARGET; + if (!raw) return Array.from(DEFAULT_TARGETS); + + const seen = new Set(); + for (const value of parseCsv(raw)) { + const normalized = normalizeTargetId(value); + if (!normalized) { + throw new Error( + `invalid sync bench target "${value}", expected one of: direct, local, remote (${ALL_TARGETS.join(", ")})` + ); + } + seen.add(normalized); + } + return seen.size > 0 ? Array.from(seen) : Array.from(DEFAULT_TARGETS); +} + +function parseStorages(argv: string[]): StorageKind[] { + const raw = + parseFlagValue(argv, "--storages") ?? + parseFlagValue(argv, "--storage") ?? + process.env.SYNC_BENCH_STORAGES ?? + process.env.SYNC_BENCH_STORAGE; + if (!raw) return Array.from(DEFAULT_STORAGES); + + const seen = new Set(); + for (const value of parseCsv(raw)) { + if (value !== "memory" && value !== "file") { + throw new Error(`invalid sync bench storage "${value}", expected one of: memory, file`); + } + seen.add(value); + } + return seen.size > 0 ? Array.from(seen) : Array.from(DEFAULT_STORAGES); +} + +function parseWorkloads(argv: string[]): SyncBenchWorkload[] { + const raw = + parseFlagValue(argv, "--workloads") ?? + parseFlagValue(argv, "--workload") ?? + process.env.SYNC_BENCH_WORKLOADS ?? + process.env.SYNC_BENCH_WORKLOAD; + if (!raw) return Array.from(DEFAULT_SYNC_BENCH_WORKLOADS); + + const seen = new Set(); + for (const value of parseCsv(raw)) { + if (!(ALL_WORKLOADS as readonly string[]).includes(value)) { + throw new Error( + `invalid sync bench workload "${value}", expected one of: ${ALL_WORKLOADS.join(", ")}` + ); + } + seen.add(value as SyncBenchWorkload); + } + return seen.size > 0 ? Array.from(seen) : Array.from(ALL_WORKLOADS); +} + +function parseFanout(argv: string[]): number { + return parsePositiveIntFlag(argv, "--fanout", "SYNC_BENCH_FANOUT", DEFAULT_SYNC_BENCH_FANOUT); +} + +function normalizeSyncServerUrl(raw: string, docId: string): URL { + let input = raw.trim(); + if (input.length === 0) throw new Error("Sync server URL is empty"); + if (!/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(input)) input = `ws://${input}`; + + const url = new URL(input); + if (url.protocol === "http:") url.protocol = "ws:"; + if (url.protocol === "https:") url.protocol = "wss:"; + if (url.protocol !== "ws:" && url.protocol !== "wss:") { + throw new Error("Sync server URL must use ws://, wss://, http://, or https://"); + } + if (url.pathname === "/" || url.pathname.length === 0) { + url.pathname = "/sync"; + } + url.searchParams.set("docId", docId); + return url; +} function hexToBytes(hex: string): Uint8Array { return nodeIdToBytes16(hex); } -function parseOpRefs(raw: any): Uint8Array[] { - return decodeSqliteOpRefs(raw); +function countOps(db: Database.Database): number { + return (db.prepare("SELECT COUNT(*) AS cnt FROM ops").get() as { cnt: number }).cnt; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function closeWebSocket(ws: WebSocket): Promise { + if (ws.readyState === WebSocket.CLOSED) return; + await new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + resolve(); + }; + ws.once("close", finish); + ws.once("error", finish); + try { + ws.close(); + } catch { + finish(); + } + setTimeout(finish, 1_000); + }); +} + +async function openWebSocket(url: URL): Promise { + return await new Promise((resolve, reject) => { + const ws = new WebSocket(url); + let settled = false; + + const finish = ( + fn: (value: WebSocket | Error) => void, + value: WebSocket | Error + ) => { + if (settled) return; + settled = true; + clearTimeout(timer); + ws.off("open", onOpen); + ws.off("error", onError); + fn(value); + }; + + const onOpen = () => finish(resolve, ws); + const onError = (error: Error) => { + void closeWebSocket(ws); + finish(reject, error); + }; + + const timer = setTimeout(() => { + void closeWebSocket(ws); + finish(reject, new Error(`timed out connecting to ${url.toString()}`)); + }, 5_000); + + ws.once("open", onOpen); + ws.once("error", onError); + }); } -function parseOps(raw: any): Operation[] { - return decodeSqliteOps(raw); +function createNodeWebSocketTransport( + ws: WebSocket +): DuplexTransport { + return { + send: async (bytes) => + await new Promise((resolve, reject) => { + ws.send(bytes, { binary: true }, (error) => { + if (!error) { + resolve(); + return; + } + reject(error instanceof Error ? error : new Error(String(error))); + }); + }), + onMessage: (handler) => { + const onMessage = (data: WebSocket.RawData) => { + if (data instanceof Uint8Array) { + handler(data); + } else if (data instanceof ArrayBuffer) { + handler(new Uint8Array(data)); + } else if (Array.isArray(data)) { + handler(Buffer.concat(data)); + } else { + handler(Buffer.from(data)); + } + }; + ws.on("message", onMessage); + return () => ws.off("message", onMessage); + }, + }; } -function makeBackend(opts: { +async function makeBackend(opts: { db: Database.Database; docId: string; initialMaxLamport: number; -}): FlushableSyncBackend { - const api = createSqliteNodeApi(opts.db); +}): Promise> { + const client = await createTreecrdtClient(opts.db, { docId: opts.docId }); + const backend = createTreecrdtSyncBackendFromClient(client, opts.docId); return makeQueuedSyncBackend({ docId: opts.docId, initialMaxLamport: opts.initialMaxLamport, maxLamportFromOps: maxLamport, - listOpRefs: async (filter) => { - if ('all' in filter) { - return parseOpRefs(await api.opRefsAll()); - } - const parent = Buffer.from(filter.children.parent); - return parseOpRefs(await api.opRefsChildren(parent)); - }, - getOpsByOpRefs: async (opRefs) => { - if (opRefs.length === 0) return []; - return parseOps(await api.opsByOpRefs(opRefs)); - }, - applyOps: async (ops) => - api.appendOps!(ops, hexToBytes, (r) => (typeof r === 'string' ? Buffer.from(r) : r)), + listOpRefs: backend.listOpRefs, + getOpsByOpRefs: backend.getOpsByOpRefs, + applyOps: backend.applyOps, }); } @@ -131,100 +380,427 @@ async function openDb(opts: { dbPath?: string; docId: string; }): Promise { - const db = new Database(opts.storage === 'memory' ? ':memory:' : (opts.dbPath ?? ':memory:')); + const db = new Database( + opts.storage === "memory" ? ":memory:" : opts.dbPath ?? ":memory:" + ); loadTreecrdtExtension(db); await createSqliteNodeApi(db).setDocId(opts.docId); return db; } -async function runBenchOnce( +async function appendInitialOps( + db: Database.Database, + ops: Operation[] +): Promise { + if (ops.length === 0) return; + const api = createSqliteNodeApi(db); + await api.appendOps!( + ops, + hexToBytes, + (replica) => (typeof replica === "string" ? Buffer.from(replica) : replica) + ); +} + +async function connectToSyncServer( + baseUrl: string, + docId: string +): Promise { + const url = normalizeSyncServerUrl(baseUrl, docId); + const ws = await openWebSocket(url); + const wire = createNodeWebSocketTransport(ws); + const transport = wrapDuplexTransportWithCodec( + wire, + treecrdtSyncV0ProtobufCodec as any + ); + return { + transport, + close: async () => { + await closeWebSocket(ws); + }, + }; +} + +async function createLocalPostgresSyncServerTarget( repoRoot: string, - { storage, workload, size }: BenchCase, - bench: ReturnType, -): Promise { - const runId = crypto.randomUUID(); - const outDir = path.join(repoRoot, 'tmp', 'sqlite-node-sync-bench'); - const dbPathA = - storage === 'file' ? path.join(outDir, `${runId}-${workload}-${size}-a.db`) : undefined; - const dbPathB = - storage === 'file' ? path.join(outDir, `${runId}-${workload}-${size}-b.db`) : undefined; - if (storage === 'file') { + postgresUrl: string +): Promise { + const port = await findFreePort(); + const backendModule = path.join( + repoRoot, + "packages", + "treecrdt-postgres-napi", + "dist", + "index.js" + ); + + const server = await startSyncServer({ + host: "127.0.0.1", + port, + postgresUrl, + backendModule, + allowDocCreate: true, + enablePgNotify: false, + }); + + return { + id: "local-postgres-sync-server", + connect: async (docId) => + await connectToSyncServer(`ws://127.0.0.1:${server.port}`, docId), + close: async () => { + await server.close(); + }, + }; +} + +async function findFreePort(): Promise { + return await new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(() => reject(new Error("failed to resolve an ephemeral port"))); + return; + } + const { port } = address; + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(port); + }); + }); + }); +} + +function createRemoteSyncServerTarget( + baseUrl: string +): SyncBenchTargetRuntime { + return { + id: "remote-sync-server", + connect: async (docId) => await connectToSyncServer(baseUrl, docId), + close: async () => {}, + }; +} + +async function prepareTargetRuntimes( + repoRoot: string, + argv: string[], + targets: SyncBenchTargetId[] +): Promise, SyncBenchTargetRuntime>> { + const runtimes = new Map< + Exclude, + SyncBenchTargetRuntime + >(); + + if (targets.includes("local-postgres-sync-server")) { + const postgresUrl = + parseFlagValue(argv, "--postgres-url") ?? process.env.TREECRDT_POSTGRES_URL; + if (!postgresUrl) { + throw new Error( + "local-postgres-sync-server target requires TREECRDT_POSTGRES_URL or --postgres-url=..." + ); + } + runtimes.set( + "local-postgres-sync-server", + await createLocalPostgresSyncServerTarget(repoRoot, postgresUrl) + ); + } + + if (targets.includes("remote-sync-server")) { + const remoteUrl = + parseFlagValue(argv, "--sync-server-url") ?? + process.env.TREECRDT_SYNC_SERVER_URL; + if (!remoteUrl) { + throw new Error( + "remote-sync-server target requires TREECRDT_SYNC_SERVER_URL or --sync-server-url=..." + ); + } + runtimes.set("remote-sync-server", createRemoteSyncServerTarget(remoteUrl)); + } + + return runtimes; +} + +async function closeTargetRuntimes( + runtimes: Map, SyncBenchTargetRuntime> +): Promise { + await Promise.allSettled(Array.from(runtimes.values(), (runtime) => runtime.close())); +} + +async function syncBackendThroughServer( + runtime: SyncBenchTargetRuntime, + docId: string, + backend: FlushableSyncBackend, + filter: Filter +): Promise { + const peer = new SyncPeer(backend, { + maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + }); + const connection = await runtime.connect(docId); + const detach = peer.attach(connection.transport); + + try { + await peer.syncOnce(connection.transport, filter, { + maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + }); + await backend.flush(); + } finally { + detach(); + await connection.close(); + } +} + +async function seedServerState( + runtime: SyncBenchTargetRuntime, + docId: string, + ops: Operation[] +): Promise { + if (ops.length === 0) return; + + const seedDb = await openDb({ storage: "memory", docId }); + try { + await appendInitialOps(seedDb, ops); + const seedBackend = await makeBackend({ + db: seedDb, + docId, + initialMaxLamport: maxLamport(ops), + }); + await syncBackendThroughServer(runtime, docId, seedBackend, { all: {} }); + } finally { + seedDb.close(); + } +} + +async function waitForServerOpCount( + runtime: SyncBenchTargetRuntime, + docId: string, + expectedCount: number +): Promise { + const deadline = Date.now() + SERVER_READY_TIMEOUT_MS; + while (true) { + const verifierDb = await openDb({ storage: "memory", docId }); + try { + const verifierBackend = await makeBackend({ + db: verifierDb, + docId, + initialMaxLamport: 0, + }); + await syncBackendThroughServer(runtime, docId, verifierBackend, { all: {} }); + if (countOps(verifierDb) === expectedCount) { + return; + } + } finally { + verifierDb.close(); + } + + if (Date.now() >= deadline) { + throw new Error( + `timed out waiting for server doc ${docId} to reach ${expectedCount} ops` + ); + } + await sleep(SERVER_READY_POLL_MS); + } +} + +async function openClientDbForRun( + repoRoot: string, + storage: StorageKind, + docId: string, + runId: string, + workload: SyncBenchWorkload, + size: number +): Promise<{ db: Database.Database; cleanup: () => Promise }> { + const outDir = path.join(repoRoot, "tmp", "sqlite-node-sync-bench"); + const dbPath = + storage === "file" + ? path.join(outDir, `${runId}-${workload}-${size}-${docId}.db`) + : undefined; + if (storage === "file") { await fs.mkdir(outDir, { recursive: true }); } + const db = await openDb({ storage, dbPath, docId }); + return { + db, + cleanup: async () => { + db.close(); + if (dbPath) { + await fs.rm(dbPath).catch(() => {}); + } + }, + }; +} + +async function runBenchOnceDirect( + repoRoot: string, + { storage, workload, size }: BenchCase, + bench: ReturnType +): Promise { + const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runId}`; - const a = await openDb({ storage, dbPath: dbPathA, docId }); - const b = await openDb({ storage, dbPath: dbPathB, docId }); + const clientA = await openClientDbForRun( + repoRoot, + storage, + docId, + `${runId}-a`, + workload, + size + ); + const clientB = await openClientDbForRun( + repoRoot, + storage, + docId, + `${runId}-b`, + workload, + size + ); try { - const opsA = bench.opsA; - const opsB = bench.opsB; - const filter = bench.filter as Filter; - - const apiA = createSqliteNodeApi(a); - const apiB = createSqliteNodeApi(b); await Promise.all([ - apiA.appendOps!(opsA, hexToBytes, (r) => (typeof r === 'string' ? Buffer.from(r) : r)), - apiB.appendOps!(opsB, hexToBytes, (r) => (typeof r === 'string' ? Buffer.from(r) : r)), + appendInitialOps(clientA.db, bench.opsA), + appendInitialOps(clientB.db, bench.opsB), ]); - const backendA = makeBackend({ db: a, docId, initialMaxLamport: maxLamport(opsA) }); - const backendB = makeBackend({ db: b, docId, initialMaxLamport: maxLamport(opsB) }); + const backendA = await makeBackend({ + db: clientA.db, + docId, + initialMaxLamport: maxLamport(bench.opsA), + }); + const backendB = await makeBackend({ + db: clientB.db, + docId, + initialMaxLamport: maxLamport(bench.opsB), + }); - const { - peerA: pa, - transportA: ta, - detach, - } = createInMemoryConnectedPeers({ + const { peerA: pa, transportA: ta, detach } = createInMemoryConnectedPeers({ backendA, backendB, codec: treecrdtSyncV0ProtobufCodec, peerOptions: { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS }, }); + try { const start = performance.now(); - await pa.syncOnce(ta, filter, { + await pa.syncOnce(ta, bench.filter as Filter, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, }); await Promise.all([backendA.flush(), backendB.flush()]); const end = performance.now(); - const countA = (a.prepare('SELECT COUNT(*) AS cnt FROM ops').get() as any).cnt as number; - const countB = (b.prepare('SELECT COUNT(*) AS cnt FROM ops').get() as any).cnt as number; - if (countA !== bench.expectedFinalOpsA || countB !== bench.expectedFinalOpsB) { + const countA = countOps(clientA.db); + const countB = countOps(clientB.db); + if ( + countA !== bench.expectedFinalOpsA || + countB !== bench.expectedFinalOpsB + ) { throw new Error( - `sync bench mismatch: expected a=${bench.expectedFinalOpsA} b=${bench.expectedFinalOpsB}, got a=${countA} b=${countB}`, + `sync bench mismatch: expected a=${bench.expectedFinalOpsA} b=${bench.expectedFinalOpsB}, got a=${countA} b=${countB}` ); } - const durationMs = end - start; - return durationMs; + return end - start; } finally { detach(); } } finally { - a.close(); - b.close(); - if (storage === 'file') { - await Promise.allSettled([ - dbPathA ? fs.rm(dbPathA) : Promise.resolve(), - dbPathB ? fs.rm(dbPathB) : Promise.resolve(), - ]); + await Promise.all([clientA.cleanup(), clientB.cleanup()]); + } +} + +async function runBenchOnceViaServer( + repoRoot: string, + runtime: SyncBenchTargetRuntime, + { storage, workload, size }: BenchCase, + bench: ReturnType +): Promise { + const runId = crypto.randomUUID(); + const docId = `sqlite-node-sync-bench-${runtime.id}-${runId}`; + const client = await openClientDbForRun( + repoRoot, + storage, + docId, + runId, + workload, + size + ); + + try { + await appendInitialOps(client.db, bench.opsA); + await seedServerState(runtime, docId, bench.opsB); + await waitForServerOpCount(runtime, docId, bench.opsB.length); + + const clientBackend = await makeBackend({ + db: client.db, + docId, + initialMaxLamport: maxLamport(bench.opsA), + }); + + const peer = new SyncPeer(clientBackend, { + maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + }); + const connection = await runtime.connect(docId); + const detach = peer.attach(connection.transport); + + try { + const start = performance.now(); + await peer.syncOnce(connection.transport, bench.filter as Filter, { + maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + }); + await clientBackend.flush(); + const end = performance.now(); + + const countA = countOps(client.db); + if (countA !== bench.expectedFinalOpsA) { + throw new Error( + `sync bench mismatch: expected client=${bench.expectedFinalOpsA}, got client=${countA}` + ); + } + + return end - start; + } finally { + detach(); + await connection.close(); } + } finally { + await client.cleanup(); } } -async function runBenchCase(repoRoot: string, benchCase: BenchCase): Promise { - const bench = buildSyncBenchCase({ workload: benchCase.workload, size: benchCase.size }); - const { size, iterations } = benchCase; +async function runBenchCase( + repoRoot: string, + benchCase: BenchCase, + runtimes: Map, SyncBenchTargetRuntime> +): Promise { + const bench = buildSyncBenchCase({ + workload: benchCase.workload, + size: benchCase.size, + fanout: benchCase.fanout, + }); + const { iterations } = benchCase; + + const runtime = + benchCase.target === "direct" ? null : runtimes.get(benchCase.target); + if (benchCase.target !== "direct" && !runtime) { + throw new Error(`missing runtime for sync bench target ${benchCase.target}`); + } const samplesMs: number[] = []; for (let i = 0; i < iterations; i += 1) { - samplesMs.push(await runBenchOnce(repoRoot, benchCase, bench)); + samplesMs.push( + runtime + ? await runBenchOnceViaServer(repoRoot, runtime, benchCase, bench) + : await runBenchOnceDirect(repoRoot, benchCase, bench) + ); } - const durationMs = iterations > 1 ? quantile(samplesMs, 0.5) : (samplesMs[0] ?? 0); + const durationMs = + iterations > 1 ? quantile(samplesMs, 0.5) : samplesMs[0] ?? 0; const opsPerSec = durationMs > 0 ? (bench.totalOps / durationMs) * 1000 : Infinity; return { @@ -234,7 +810,17 @@ async function runBenchCase(repoRoot: string, benchCase: BenchCase): Promise 1 ? iterations : undefined, @@ -253,37 +839,49 @@ async function main() { const config = parseConfigFromArgv(argv) ?? [...SYNC_BENCH_CONFIG]; const rootConfig = [...SYNC_BENCH_ROOT_CONFIG]; + const targets = parseTargets(argv); + const storages = parseStorages(argv); + const workloads = parseWorkloads(argv); + const fanout = parseFanout(argv); + const runtimes = await prepareTargetRuntimes(repoRoot, argv, targets); - const cases: BenchCase[] = []; - for (const storage of ['memory', 'file'] as const) { - for (const workload of DEFAULT_SYNC_BENCH_WORKLOADS) { - for (const [size, iterations] of config) { - cases.push({ storage, workload, size, iterations }); - } - } - for (const workload of DEFAULT_SYNC_BENCH_ROOT_CHILDREN_WORKLOADS) { - for (const [size, iterations] of rootConfig) { - cases.push({ storage, workload, size, iterations }); + try { + const cases: BenchCase[] = []; + for (const target of targets) { + for (const storage of storages) { + for (const workload of workloads) { + const entries = + workload === "sync-root-children-fanout10" ? rootConfig : config; + for (const [size, iterations] of entries) { + cases.push({ target, storage, workload, size, iterations, fanout }); + } + } } } - } - for (const benchCase of cases) { - const result = await runBenchCase(repoRoot, benchCase); - const outFile = path.join( - repoRoot, - 'benchmarks', - 'sqlite-node-sync', - `${benchCase.storage}-${result.name}.json`, - ); - const payload = await writeResult(result, { - implementation: 'sqlite-node', - storage: benchCase.storage, - workload: result.name, - outFile, - extra: { count: benchCase.size, ...result.extra }, - }); - console.log(JSON.stringify(payload)); + for (const benchCase of cases) { + const result = await runBenchCase(repoRoot, benchCase, runtimes); + const outFile = path.join( + repoRoot, + "benchmarks", + "sqlite-node-sync", + `${benchCase.storage}-${benchCase.target}-${result.name}.json` + ); + const payload = await writeResult(result, { + implementation: "sqlite-node", + storage: benchCase.storage, + workload: result.name, + outFile, + extra: { + count: benchCase.size, + target: benchCase.target, + ...result.extra, + }, + }); + console.log(JSON.stringify(payload)); + } + } finally { + await closeTargetRuntimes(runtimes); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1424f1e1..495fa994 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -368,18 +368,30 @@ importers: '@treecrdt/benchmark': specifier: workspace:* version: link:../treecrdt-benchmark + '@treecrdt/postgres-napi': + specifier: workspace:* + version: link:../treecrdt-postgres-napi '@treecrdt/sqlite-conformance': specifier: workspace:* version: link:../treecrdt-sqlite-conformance '@treecrdt/sync': specifier: workspace:* version: link:../sync/protocol + '@treecrdt/sync-server-postgres-node': + specifier: workspace:* + version: link:../sync/server/postgres-node + '@treecrdt/sync-sqlite': + specifier: workspace:* + version: link:../sync/material/sqlite '@types/better-sqlite3': specifier: ^7.6.12 version: 7.6.13 '@types/node': specifier: ^20.19.25 version: 20.19.25 + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 better-sqlite3: specifier: ^9.0.0 version: 9.6.0 @@ -395,6 +407,9 @@ importers: vitest: specifier: ^1.6.0 version: 1.6.1(@types/node@20.19.25) + ws: + specifier: ^8.18.3 + version: 8.19.0 packages/treecrdt-ts: devDependencies: diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs new file mode 100644 index 00000000..f00b5230 --- /dev/null +++ b/scripts/run-sync-bench.mjs @@ -0,0 +1,181 @@ +import { spawn } from "node:child_process"; + +import { repoRootFromImportMeta } from "./repo-root.mjs"; + +const repoRoot = repoRootFromImportMeta(import.meta.url, 1); +const LOCAL_POSTGRES_URL = "postgres://postgres:postgres@127.0.0.1:5432/postgres"; + +function normalizeTarget(raw) { + const value = raw.trim(); + if (value === "direct") return "direct"; + if ( + value === "local" || + value === "local-server" || + value === "local-postgres" || + value === "local-postgres-sync-server" + ) { + return "local-postgres-sync-server"; + } + if ( + value === "remote" || + value === "remote-server" || + value === "remote-sync" || + value === "remote-sync-server" + ) { + return "remote-sync-server"; + } + return null; +} + +function parseCsv(raw) { + if (!raw) return []; + return raw + .split(",") + .map((value) => value.trim()) + .filter((value) => value.length > 0); +} + +function extractFlagValue(args, flag) { + const prefix = `${flag}=`; + const found = args.find((arg) => arg.startsWith(prefix)); + return found ? found.slice(prefix.length).trim() : undefined; +} + +function parseRequestedTargets(args) { + const forwardedTargets = + extractFlagValue(args, "--targets") ?? extractFlagValue(args, "--target"); + if (forwardedTargets) { + const normalized = parseCsv(forwardedTargets) + .map((value) => normalizeTarget(value)) + .filter(Boolean); + if (normalized.length > 0) return normalized; + } + + const positional = args.find((arg) => !arg.startsWith("--")); + if (positional) { + const normalized = normalizeTarget(positional); + if (normalized) return [normalized]; + } + + return ["direct"]; +} + +function stripPositionalTargetArg(args) { + let removed = false; + return args.filter((arg) => { + if (removed || arg.startsWith("--")) return true; + const normalized = normalizeTarget(arg); + if (!normalized) return true; + removed = true; + return false; + }); +} + +function hasFlag(args, flag) { + return args.some((arg) => arg.startsWith(`${flag}=`)); +} + +async function runPnpm(args, env) { + await new Promise((resolve, reject) => { + const child = spawn("pnpm", args, { + cwd: repoRoot, + env, + stdio: "inherit", + }); + child.on("exit", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + reject( + new Error( + `pnpm ${args.join(" ")} failed with ${signal ? `signal ${signal}` : `exit code ${code}`}` + ) + ); + }); + child.on("error", reject); + }); +} + +function ensureRemoteUrl(args, env) { + if (hasFlag(args, "--sync-server-url")) return; + if (env.TREECRDT_SYNC_SERVER_URL?.trim()) return; + throw new Error( + "remote sync bench requires TREECRDT_SYNC_SERVER_URL or --sync-server-url=ws://host/sync (use wss:// for public TLS deployments)" + ); +} + +function effectiveEnv(targets, args) { + const env = { ...process.env }; + if ( + targets.includes("local-postgres-sync-server") && + !hasFlag(args, "--postgres-url") && + !(env.TREECRDT_POSTGRES_URL && env.TREECRDT_POSTGRES_URL.trim()) + ) { + env.TREECRDT_POSTGRES_URL = LOCAL_POSTGRES_URL; + } + if (targets.includes("remote-sync-server")) { + ensureRemoteUrl(args, env); + } + return env; +} + +async function main() { + const rawArgs = process.argv.slice(2); + if (rawArgs.includes("--help")) { + console.log(`Usage: + pnpm benchmark:sync + pnpm benchmark:sync:direct -- --workloads=sync-balanced-children-payloads-cold-start --counts=10000,50000 --fanout=10 + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 + TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start + +Notes: + - local sync benches default TREECRDT_POSTGRES_URL to ${LOCAL_POSTGRES_URL} + - remote sync benches never hardcode a server URL; pass TREECRDT_SYNC_SERVER_URL or --sync-server-url=... + - use wss:// for public HTTPS/TLS deployments and ws:// for local/plain HTTP servers + - use --fanout=20 to model broader trees; default fanout is 10 + - extra args are forwarded to packages/treecrdt-sqlite-node/scripts/bench-sync.ts`); + return; + } + + const targets = parseRequestedTargets(rawArgs); + const forwardedArgs = stripPositionalTargetArg(rawArgs); + const env = effectiveEnv(targets, forwardedArgs); + + const buildCommands = [ + ["-C", "packages/treecrdt-benchmark", "run", "build"], + ["-C", "packages/sync/protocol", "run", "build"], + ["-C", "packages/sync/material/sqlite", "run", "build"], + ["-C", "packages/sync/server/postgres-node", "run", "build"], + ["-C", "packages/treecrdt-sqlite-node", "run", "build"], + ]; + if (targets.includes("local-postgres-sync-server")) { + buildCommands.splice(3, 0, [ + "-C", + "packages/treecrdt-postgres-napi", + "run", + "build", + ]); + } + + for (const command of buildCommands) { + await runPnpm(command, env); + } + + const targetFlagPresent = hasFlag(forwardedArgs, "--targets") || hasFlag(forwardedArgs, "--target"); + const benchArgs = [ + "-C", + "packages/treecrdt-sqlite-node", + "exec", + "tsx", + "./scripts/bench-sync.ts", + ...(targetFlagPresent ? [] : [`--targets=${targets.join(",")}`]), + ...forwardedArgs, + ]; + await runPnpm(benchArgs, env); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); From 4ebbb8f414ac14839f21dcce3445eb13ff9f39fc Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Fri, 20 Mar 2026 15:12:10 +0100 Subject: [PATCH 02/58] feat(benchmark): add sync time-to-first-view metric --- docs/BENCHMARKS.md | 37 ++++-- packages/treecrdt-benchmark/src/sync.ts | 14 ++ .../scripts/bench-sync.ts | 124 +++++++++++++++--- scripts/run-sync-bench.mjs | 2 + 4 files changed, 148 insertions(+), 29 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index b422220b..c6cd8f1f 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -33,6 +33,7 @@ pnpm benchmark:postgres - First view on a new device, structure only: `benchmark:sync:*` with `sync-balanced-children-cold-start` - First view on a new device, with payloads: `benchmark:sync:*` with `sync-balanced-children-payloads-cold-start` +- Single end-to-end time-to-first-visible-page number: `benchmark:sync:*` with the same balanced workloads plus `--first-view` - Local render cost after the data is already present: `benchmark:sqlite-node:note-paths -- --benches=read-children-payloads` - Local mutation cost inside a large existing tree: `benchmark:sqlite-node:note-paths -- --benches=insert-into-large-tree` - Protocol/storage baselines and worst-case stress: `sync-one-missing`, `sync-all`, `sync-children*`, `sync-root-children-fanout10` @@ -74,6 +75,22 @@ pnpm benchmark:sync:remote -- \ Use `--fanout=20` when you want to model a broader notebook tree. +### Time To First Visible Page + +Add `--first-view` when you want one number that includes: + +- scoped sync into the local store +- the immediate local `childrenPage(...)` read +- payload fetches for the parent and visible children when the workload carries payloads + +```sh +pnpm benchmark:sync:local -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --counts=10000 \ + --fanout=10 \ + --first-view +``` + ### Local First View Read Path This measures the app-shaped local read immediately after sync: fetch the visible children page plus payloads for the parent and those children. @@ -117,6 +134,13 @@ postgres://postgres:postgres@127.0.0.1:5432/postgres Override with `TREECRDT_POSTGRES_URL` or `--postgres-url=...`. +The Docker helper is only a convenience. The local sync benchmark just needs a reachable Postgres URL, so a native local Postgres instance works too: + +```sh +TREECRDT_POSTGRES_URL=postgres://postgres:postgres@127.0.0.1:55432/postgres \ +pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 +``` + ### Remote Sync Server URL The remote URL is intentionally not hardcoded in this repo. Different deployments can have different latency, auth, retention, and scaling settings, so pass it at runtime through `TREECRDT_SYNC_SERVER_URL` or `--sync-server-url=...`. @@ -156,6 +180,7 @@ Common sync flags: - `--storages=memory,file` - `--targets=direct,local-postgres-sync-server` - `--fanout=10` +- `--first-view` - `--sync-server-url=ws://host/sync` - `--postgres-url=postgres://...` @@ -169,13 +194,7 @@ Common note-path flags: ## What Is Still Missing? -The suite is much closer to real note-taking behavior now, but there is still one gap worth closing later: - -- a single end-to-end benchmark that measures cold-start sync and the first local render in one number - -Right now that path is covered in two pieces: - -- sync time via `sync-balanced-children*-cold-start` -- local read time via `read-children-payloads` +The remaining gaps are mostly infrastructure-related now: -That is already enough to identify where time is going, but an integrated "time to first visible page" benchmark would still be useful as a final product metric. +- a healthy, repeatable local Postgres bootstrap path that does not depend on a stuck Docker daemon +- a working public websocket deployment path for the remote sync target diff --git a/packages/treecrdt-benchmark/src/sync.ts b/packages/treecrdt-benchmark/src/sync.ts index 01796dee..55f5c1e8 100644 --- a/packages/treecrdt-benchmark/src/sync.ts +++ b/packages/treecrdt-benchmark/src/sync.ts @@ -73,6 +73,13 @@ export type SyncBenchCase = { extra: Record; expectedFinalOpsA: number; expectedFinalOpsB: number; + firstView?: { + parent: string; + pageSize: number; + expectedChildren: number; + includePayloads: boolean; + payloadBytes?: number; + }; }; export function nodeIdFromInt(i: number): string { @@ -225,6 +232,13 @@ function buildBalancedChildrenColdStartCase(opts: { }, expectedFinalOpsA: opsA.length + transferredOps, expectedFinalOpsB: opsB.length, + firstView: { + parent: targetParent, + pageSize: Math.min(DEFAULT_SYNC_BENCH_PAGE_SIZE, targetChildren.length), + expectedChildren: targetChildren.length, + includePayloads: opts.withPayloads, + payloadBytes: opts.withPayloads ? opts.payloadBytes : undefined, + }, }; } diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index c9fa3c27..d3b26294 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -63,6 +63,12 @@ type SyncBenchResult = { extra?: Record; }; +type SyncBenchSample = { + totalMs: number; + syncMs: number; + firstViewReadMs: number; +}; + type SyncBenchConnection = { transport: DuplexTransport; close: () => Promise; @@ -149,6 +155,13 @@ function parsePositiveIntFlag(argv: string[], flag: string, envName: string, fal return value; } +function parseBooleanFlag(argv: string[], flag: string, envName: string): boolean { + const envRaw = process.env[envName]; + if (argv.includes(flag)) return true; + if (envRaw == null || envRaw === "") return false; + return ["1", "true", "yes", "on"].includes(envRaw.trim().toLowerCase()); +} + function parseCsv(raw: string | undefined): string[] { if (!raw) return []; return raw @@ -242,6 +255,10 @@ function parseFanout(argv: string[]): number { return parsePositiveIntFlag(argv, "--fanout", "SYNC_BENCH_FANOUT", DEFAULT_SYNC_BENCH_FANOUT); } +function parseFirstView(argv: string[]): boolean { + return parseBooleanFlag(argv, "--first-view", "SYNC_BENCH_FIRST_VIEW"); +} + function normalizeSyncServerUrl(raw: string, docId: string): URL { let input = raw.trim(); if (input.length === 0) throw new Error("Sync server URL is empty"); @@ -401,6 +418,33 @@ async function appendInitialOps( ); } +async function measureFirstViewAfterSync( + db: Database.Database, + docId: string, + firstView: NonNullable["firstView"]> +): Promise { + const client = await createTreecrdtClient(db, { docId }); + const expectedChildren = Math.min(firstView.expectedChildren, firstView.pageSize); + const startedAt = performance.now(); + const rows = await client.tree.childrenPage(firstView.parent, null, firstView.pageSize); + if (!Array.isArray(rows) || rows.length !== expectedChildren) { + throw new Error(`expected ${expectedChildren} child rows after sync, got ${Array.isArray(rows) ? rows.length : "non-array"}`); + } + + if (firstView.includePayloads) { + const parentPayload = await client.tree.getPayload(firstView.parent); + if (!(parentPayload instanceof Uint8Array) || parentPayload.length !== firstView.payloadBytes) { + throw new Error("expected scope-root payload to be present after sync"); + } + const payloads = await Promise.all(rows.map((row: { node: string }) => client.tree.getPayload(row.node))); + if (payloads.some((payload) => !(payload instanceof Uint8Array) || payload.length !== firstView.payloadBytes)) { + throw new Error("expected all first-view child payloads to be present after sync"); + } + } + + return performance.now() - startedAt; +} + async function connectToSyncServer( baseUrl: string, docId: string @@ -638,8 +682,9 @@ async function openClientDbForRun( async function runBenchOnceDirect( repoRoot: string, { storage, workload, size }: BenchCase, - bench: ReturnType -): Promise { + bench: ReturnType, + includeFirstView: boolean +): Promise { const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runId}`; const clientA = await openClientDbForRun( @@ -690,7 +735,15 @@ async function runBenchOnceDirect( codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, }); await Promise.all([backendA.flush(), backendB.flush()]); - const end = performance.now(); + const syncedAt = performance.now(); + + let firstViewReadMs = 0; + if (includeFirstView) { + if (!bench.firstView) { + throw new Error(`sync bench workload ${bench.name} does not define a first-view read path`); + } + firstViewReadMs = await measureFirstViewAfterSync(clientA.db, docId, bench.firstView); + } const countA = countOps(clientA.db); const countB = countOps(clientB.db); @@ -703,7 +756,11 @@ async function runBenchOnceDirect( ); } - return end - start; + return { + totalMs: syncedAt - start + firstViewReadMs, + syncMs: syncedAt - start, + firstViewReadMs, + }; } finally { detach(); } @@ -716,8 +773,9 @@ async function runBenchOnceViaServer( repoRoot: string, runtime: SyncBenchTargetRuntime, { storage, workload, size }: BenchCase, - bench: ReturnType -): Promise { + bench: ReturnType, + includeFirstView: boolean +): Promise { const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runtime.id}-${runId}`; const client = await openClientDbForRun( @@ -753,7 +811,15 @@ async function runBenchOnceViaServer( codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, }); await clientBackend.flush(); - const end = performance.now(); + const syncedAt = performance.now(); + + let firstViewReadMs = 0; + if (includeFirstView) { + if (!bench.firstView) { + throw new Error(`sync bench workload ${bench.name} does not define a first-view read path`); + } + firstViewReadMs = await measureFirstViewAfterSync(client.db, docId, bench.firstView); + } const countA = countOps(client.db); if (countA !== bench.expectedFinalOpsA) { @@ -762,7 +828,11 @@ async function runBenchOnceViaServer( ); } - return end - start; + return { + totalMs: syncedAt - start + firstViewReadMs, + syncMs: syncedAt - start, + firstViewReadMs, + }; } finally { detach(); await connection.close(); @@ -775,7 +845,8 @@ async function runBenchOnceViaServer( async function runBenchCase( repoRoot: string, benchCase: BenchCase, - runtimes: Map, SyncBenchTargetRuntime> + runtimes: Map, SyncBenchTargetRuntime>, + includeFirstView: boolean ): Promise { const bench = buildSyncBenchCase({ workload: benchCase.workload, @@ -790,21 +861,28 @@ async function runBenchCase( throw new Error(`missing runtime for sync bench target ${benchCase.target}`); } - const samplesMs: number[] = []; + if (includeFirstView && !bench.firstView) { + throw new Error(`sync bench workload ${bench.name} does not support --first-view`); + } + + const samples: SyncBenchSample[] = []; for (let i = 0; i < iterations; i += 1) { - samplesMs.push( + samples.push( runtime - ? await runBenchOnceViaServer(repoRoot, runtime, benchCase, bench) - : await runBenchOnceDirect(repoRoot, benchCase, bench) + ? await runBenchOnceViaServer(repoRoot, runtime, benchCase, bench, includeFirstView) + : await runBenchOnceDirect(repoRoot, benchCase, bench, includeFirstView) ); } + const totalSamplesMs = samples.map((sample) => sample.totalMs); + const syncSamplesMs = samples.map((sample) => sample.syncMs); + const firstViewReadSamplesMs = samples.map((sample) => sample.firstViewReadMs); const durationMs = - iterations > 1 ? quantile(samplesMs, 0.5) : samplesMs[0] ?? 0; + iterations > 1 ? quantile(totalSamplesMs, 0.5) : totalSamplesMs[0] ?? 0; const opsPerSec = durationMs > 0 ? (bench.totalOps / durationMs) * 1000 : Infinity; return { - name: bench.name, + name: includeFirstView ? `${bench.name}-first-view` : bench.name, totalOps: bench.totalOps, durationMs, opsPerSec, @@ -821,14 +899,19 @@ async function runBenchCase( : benchCase.target === "remote-sync-server" ? "remote" : "none", + measurement: includeFirstView ? "time-to-first-view" : "sync-only", codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, iterations: iterations > 1 ? iterations : undefined, avgDurationMs: iterations > 1 ? durationMs : undefined, - samplesMs, - p95Ms: quantile(samplesMs, 0.95), - minMs: Math.min(...samplesMs), - maxMs: Math.max(...samplesMs), + samplesMs: totalSamplesMs, + syncSamplesMs, + firstViewReadSamplesMs: includeFirstView ? firstViewReadSamplesMs : undefined, + syncMedianMs: quantile(syncSamplesMs, 0.5), + firstViewReadMedianMs: includeFirstView ? quantile(firstViewReadSamplesMs, 0.5) : undefined, + p95Ms: quantile(totalSamplesMs, 0.95), + minMs: Math.min(...totalSamplesMs), + maxMs: Math.max(...totalSamplesMs), }, }; } @@ -843,6 +926,7 @@ async function main() { const storages = parseStorages(argv); const workloads = parseWorkloads(argv); const fanout = parseFanout(argv); + const includeFirstView = parseFirstView(argv); const runtimes = await prepareTargetRuntimes(repoRoot, argv, targets); try { @@ -860,7 +944,7 @@ async function main() { } for (const benchCase of cases) { - const result = await runBenchCase(repoRoot, benchCase, runtimes); + const result = await runBenchCase(repoRoot, benchCase, runtimes, includeFirstView); const outFile = path.join( repoRoot, "benchmarks", diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index f00b5230..fb00ca0f 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -127,6 +127,7 @@ async function main() { pnpm benchmark:sync pnpm benchmark:sync:direct -- --workloads=sync-balanced-children-payloads-cold-start --counts=10000,50000 --fanout=10 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start Notes: @@ -134,6 +135,7 @@ Notes: - remote sync benches never hardcode a server URL; pass TREECRDT_SYNC_SERVER_URL or --sync-server-url=... - use wss:// for public HTTPS/TLS deployments and ws:// for local/plain HTTP servers - use --fanout=20 to model broader trees; default fanout is 10 + - add --first-view to include the immediate local read after sync in the measured duration - extra args are forwarded to packages/treecrdt-sqlite-node/scripts/bench-sync.ts`); return; } From 845bee6bc43144701bf362f0d3123c7f0b92170f Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Fri, 20 Mar 2026 15:41:06 +0100 Subject: [PATCH 03/58] feat(benchmark): profile sync backend calls --- docs/BENCHMARKS.md | 20 +++ .../scripts/backend-profiler.mjs | 151 ++++++++++++++++++ .../scripts/bench-sync.ts | 79 ++++++--- .../instrumented-postgres-backend-module.mjs | 15 ++ scripts/run-sync-bench.mjs | 2 + 5 files changed, 248 insertions(+), 19 deletions(-) create mode 100644 packages/treecrdt-sqlite-node/scripts/backend-profiler.mjs create mode 100644 packages/treecrdt-sqlite-node/scripts/instrumented-postgres-backend-module.mjs diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index c6cd8f1f..edc373a3 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -91,6 +91,25 @@ pnpm benchmark:sync:local -- \ --first-view ``` +### Backend Call Profiling + +Add `--profile-backend` when you want per-backend timings for: + +- `listOpRefs` +- `getOpsByOpRefs` +- `applyOps` + +This is especially useful on the local Postgres sync-server target because it shows whether the bottleneck is on the client SQLite side or the server Postgres side. + +```sh +TREECRDT_POSTGRES_URL=postgres://postgres:postgres@127.0.0.1:55432/postgres \ +pnpm benchmark:sync:local -- \ + --workloads=sync-balanced-children-cold-start,sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --fanout=10 \ + --profile-backend +``` + ### Local First View Read Path This measures the app-shaped local read immediately after sync: fetch the visible children page plus payloads for the parent and those children. @@ -181,6 +200,7 @@ Common sync flags: - `--targets=direct,local-postgres-sync-server` - `--fanout=10` - `--first-view` +- `--profile-backend` - `--sync-server-url=ws://host/sync` - `--postgres-url=postgres://...` diff --git a/packages/treecrdt-sqlite-node/scripts/backend-profiler.mjs b/packages/treecrdt-sqlite-node/scripts/backend-profiler.mjs new file mode 100644 index 00000000..0fe8f8a9 --- /dev/null +++ b/packages/treecrdt-sqlite-node/scripts/backend-profiler.mjs @@ -0,0 +1,151 @@ +const REGISTRY_KEY = Symbol.for("treecrdt.sync.backendProfiler"); + +function registry() { + if (!globalThis[REGISTRY_KEY]) { + globalThis[REGISTRY_KEY] = new Map(); + } + return globalThis[REGISTRY_KEY]; +} + +function emptyMethodStats() { + return { + calls: 0, + errors: 0, + totalMs: 0, + maxMs: 0, + inputItems: 0, + outputItems: 0, + }; +} + +function emptyLabelStats() { + return { + listOpRefs: emptyMethodStats(), + getOpsByOpRefs: emptyMethodStats(), + applyOps: emptyMethodStats(), + }; +} + +function ensureDocStats(docId) { + const docs = registry(); + let docStats = docs.get(docId); + if (!docStats) { + docStats = new Map(); + docs.set(docId, docStats); + } + return docStats; +} + +function ensureLabelStats(docId, label) { + const docStats = ensureDocStats(docId); + let labelStats = docStats.get(label); + if (!labelStats) { + labelStats = emptyLabelStats(); + docStats.set(label, labelStats); + } + return labelStats; +} + +function finalizeMethodStats(stats) { + return { + ...stats, + avgMs: stats.calls > 0 ? stats.totalMs / stats.calls : 0, + avgInputItems: stats.calls > 0 ? stats.inputItems / stats.calls : 0, + avgOutputItems: stats.calls > 0 ? stats.outputItems / stats.calls : 0, + }; +} + +function record(docId, label, method, durationMs, inputItems, outputItems, ok) { + const labelStats = ensureLabelStats(docId, label); + const stats = labelStats[method]; + stats.calls += 1; + stats.totalMs += durationMs; + stats.maxMs = Math.max(stats.maxMs, durationMs); + stats.inputItems += inputItems; + stats.outputItems += outputItems; + if (!ok) stats.errors += 1; +} + +function countListOpRefsOutput(value) { + return Array.isArray(value) ? value.length : 0; +} + +function countGetOpsByOpRefsOutput(value) { + return Array.isArray(value) ? value.length : 0; +} + +export function resetBackendProfiler(docId) { + if (typeof docId === "string" && docId.length > 0) { + registry().delete(docId); + return; + } + registry().clear(); +} + +export function takeBackendProfilerSnapshot(docId) { + const docStats = registry().get(docId); + if (!docStats) return null; + + const labels = {}; + for (const [label, stats] of docStats.entries()) { + labels[label] = { + listOpRefs: finalizeMethodStats(stats.listOpRefs), + getOpsByOpRefs: finalizeMethodStats(stats.getOpsByOpRefs), + applyOps: finalizeMethodStats(stats.applyOps), + }; + } + return { docId, labels }; +} + +export function wrapBackendWithProfiler(backend, { docId, label }) { + if (!backend || typeof backend !== "object") return backend; + if (typeof docId !== "string" || docId.length === 0) return backend; + if (typeof label !== "string" || label.length === 0) return backend; + + return { + ...backend, + listOpRefs: async (filter) => { + const startedAt = performance.now(); + try { + const result = await backend.listOpRefs(filter); + record(docId, label, "listOpRefs", performance.now() - startedAt, 1, countListOpRefsOutput(result), true); + return result; + } catch (error) { + record(docId, label, "listOpRefs", performance.now() - startedAt, 1, 0, false); + throw error; + } + }, + getOpsByOpRefs: async (opRefs) => { + const inputItems = Array.isArray(opRefs) ? opRefs.length : 0; + const startedAt = performance.now(); + try { + const result = await backend.getOpsByOpRefs(opRefs); + record( + docId, + label, + "getOpsByOpRefs", + performance.now() - startedAt, + inputItems, + countGetOpsByOpRefsOutput(result), + true + ); + return result; + } catch (error) { + record(docId, label, "getOpsByOpRefs", performance.now() - startedAt, inputItems, 0, false); + throw error; + } + }, + applyOps: async (ops) => { + const inputItems = Array.isArray(ops) ? ops.length : 0; + const startedAt = performance.now(); + try { + const result = await backend.applyOps(ops); + record(docId, label, "applyOps", performance.now() - startedAt, inputItems, 0, true); + return result; + } catch (error) { + record(docId, label, "applyOps", performance.now() - startedAt, inputItems, 0, false); + throw error; + } + }, + }; +} diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index d3b26294..da487e77 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -38,6 +38,11 @@ import { createSqliteNodeApi, loadTreecrdtExtension, } from "../dist/index.js"; +import { + resetBackendProfiler, + takeBackendProfilerSnapshot, + wrapBackendWithProfiler, +} from "./backend-profiler.mjs"; type StorageKind = "memory" | "file"; type ConfigEntry = [number, number]; @@ -67,6 +72,7 @@ type SyncBenchSample = { totalMs: number; syncMs: number; firstViewReadMs: number; + backendProfile?: unknown; }; type SyncBenchConnection = { @@ -259,6 +265,10 @@ function parseFirstView(argv: string[]): boolean { return parseBooleanFlag(argv, "--first-view", "SYNC_BENCH_FIRST_VIEW"); } +function parseProfileBackend(argv: string[]): boolean { + return parseBooleanFlag(argv, "--profile-backend", "SYNC_BENCH_PROFILE_BACKEND"); +} + function normalizeSyncServerUrl(raw: string, docId: string): URL { let input = raw.trim(); if (input.length === 0) throw new Error("Sync server URL is empty"); @@ -378,11 +388,11 @@ async function makeBackend(opts: { db: Database.Database; docId: string; initialMaxLamport: number; + profileLabel?: string; }): Promise> { const client = await createTreecrdtClient(opts.db, { docId: opts.docId }); const backend = createTreecrdtSyncBackendFromClient(client, opts.docId); - - return makeQueuedSyncBackend({ + const queued = makeQueuedSyncBackend({ docId: opts.docId, initialMaxLamport: opts.initialMaxLamport, maxLamportFromOps: maxLamport, @@ -390,6 +400,11 @@ async function makeBackend(opts: { getOpsByOpRefs: backend.getOpsByOpRefs, applyOps: backend.applyOps, }); + if (!opts.profileLabel) return queued; + return wrapBackendWithProfiler(queued, { + docId: opts.docId, + label: opts.profileLabel, + }) as FlushableSyncBackend; } async function openDb(opts: { @@ -466,16 +481,25 @@ async function connectToSyncServer( async function createLocalPostgresSyncServerTarget( repoRoot: string, - postgresUrl: string + postgresUrl: string, + profileBackend: boolean ): Promise { const port = await findFreePort(); - const backendModule = path.join( - repoRoot, - "packages", - "treecrdt-postgres-napi", - "dist", - "index.js" - ); + const backendModule = profileBackend + ? path.join( + repoRoot, + "packages", + "treecrdt-sqlite-node", + "scripts", + "instrumented-postgres-backend-module.mjs" + ) + : path.join( + repoRoot, + "packages", + "treecrdt-postgres-napi", + "dist", + "index.js" + ); const server = await startSyncServer({ host: "127.0.0.1", @@ -532,7 +556,8 @@ function createRemoteSyncServerTarget( async function prepareTargetRuntimes( repoRoot: string, argv: string[], - targets: SyncBenchTargetId[] + targets: SyncBenchTargetId[], + profileBackend: boolean ): Promise, SyncBenchTargetRuntime>> { const runtimes = new Map< Exclude, @@ -549,7 +574,7 @@ async function prepareTargetRuntimes( } runtimes.set( "local-postgres-sync-server", - await createLocalPostgresSyncServerTarget(repoRoot, postgresUrl) + await createLocalPostgresSyncServerTarget(repoRoot, postgresUrl, profileBackend) ); } @@ -683,7 +708,8 @@ async function runBenchOnceDirect( repoRoot: string, { storage, workload, size }: BenchCase, bench: ReturnType, - includeFirstView: boolean + includeFirstView: boolean, + profileBackend: boolean ): Promise { const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runId}`; @@ -714,11 +740,13 @@ async function runBenchOnceDirect( db: clientA.db, docId, initialMaxLamport: maxLamport(bench.opsA), + profileLabel: profileBackend ? "direct-client-a" : undefined, }); const backendB = await makeBackend({ db: clientB.db, docId, initialMaxLamport: maxLamport(bench.opsB), + profileLabel: profileBackend ? "direct-client-b" : undefined, }); const { peerA: pa, transportA: ta, detach } = createInMemoryConnectedPeers({ @@ -729,6 +757,7 @@ async function runBenchOnceDirect( }); try { + if (profileBackend) resetBackendProfiler(docId); const start = performance.now(); await pa.syncOnce(ta, bench.filter as Filter, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, @@ -760,6 +789,7 @@ async function runBenchOnceDirect( totalMs: syncedAt - start + firstViewReadMs, syncMs: syncedAt - start, firstViewReadMs, + backendProfile: profileBackend ? takeBackendProfilerSnapshot(docId) : undefined, }; } finally { detach(); @@ -774,7 +804,8 @@ async function runBenchOnceViaServer( runtime: SyncBenchTargetRuntime, { storage, workload, size }: BenchCase, bench: ReturnType, - includeFirstView: boolean + includeFirstView: boolean, + profileBackend: boolean ): Promise { const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runtime.id}-${runId}`; @@ -796,6 +827,7 @@ async function runBenchOnceViaServer( db: client.db, docId, initialMaxLamport: maxLamport(bench.opsA), + profileLabel: profileBackend ? "client-sqlite" : undefined, }); const peer = new SyncPeer(clientBackend, { @@ -805,6 +837,7 @@ async function runBenchOnceViaServer( const detach = peer.attach(connection.transport); try { + if (profileBackend) resetBackendProfiler(docId); const start = performance.now(); await peer.syncOnce(connection.transport, bench.filter as Filter, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, @@ -832,6 +865,7 @@ async function runBenchOnceViaServer( totalMs: syncedAt - start + firstViewReadMs, syncMs: syncedAt - start, firstViewReadMs, + backendProfile: profileBackend ? takeBackendProfilerSnapshot(docId) : undefined, }; } finally { detach(); @@ -846,7 +880,8 @@ async function runBenchCase( repoRoot: string, benchCase: BenchCase, runtimes: Map, SyncBenchTargetRuntime>, - includeFirstView: boolean + includeFirstView: boolean, + profileBackend: boolean ): Promise { const bench = buildSyncBenchCase({ workload: benchCase.workload, @@ -869,14 +904,17 @@ async function runBenchCase( for (let i = 0; i < iterations; i += 1) { samples.push( runtime - ? await runBenchOnceViaServer(repoRoot, runtime, benchCase, bench, includeFirstView) - : await runBenchOnceDirect(repoRoot, benchCase, bench, includeFirstView) + ? await runBenchOnceViaServer(repoRoot, runtime, benchCase, bench, includeFirstView, profileBackend) + : await runBenchOnceDirect(repoRoot, benchCase, bench, includeFirstView, profileBackend) ); } const totalSamplesMs = samples.map((sample) => sample.totalMs); const syncSamplesMs = samples.map((sample) => sample.syncMs); const firstViewReadSamplesMs = samples.map((sample) => sample.firstViewReadMs); + const backendProfiles = samples + .map((sample) => sample.backendProfile) + .filter((sample): sample is NonNullable => sample != null); const durationMs = iterations > 1 ? quantile(totalSamplesMs, 0.5) : totalSamplesMs[0] ?? 0; const opsPerSec = durationMs > 0 ? (bench.totalOps / durationMs) * 1000 : Infinity; @@ -900,6 +938,8 @@ async function runBenchCase( ? "remote" : "none", measurement: includeFirstView ? "time-to-first-view" : "sync-only", + backendProfile: profileBackend ? backendProfiles.at(-1) : undefined, + backendProfileSamples: profileBackend && backendProfiles.length > 1 ? backendProfiles : undefined, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, iterations: iterations > 1 ? iterations : undefined, @@ -927,7 +967,8 @@ async function main() { const workloads = parseWorkloads(argv); const fanout = parseFanout(argv); const includeFirstView = parseFirstView(argv); - const runtimes = await prepareTargetRuntimes(repoRoot, argv, targets); + const profileBackend = parseProfileBackend(argv); + const runtimes = await prepareTargetRuntimes(repoRoot, argv, targets, profileBackend); try { const cases: BenchCase[] = []; @@ -944,7 +985,7 @@ async function main() { } for (const benchCase of cases) { - const result = await runBenchCase(repoRoot, benchCase, runtimes, includeFirstView); + const result = await runBenchCase(repoRoot, benchCase, runtimes, includeFirstView, profileBackend); const outFile = path.join( repoRoot, "benchmarks", diff --git a/packages/treecrdt-sqlite-node/scripts/instrumented-postgres-backend-module.mjs b/packages/treecrdt-sqlite-node/scripts/instrumented-postgres-backend-module.mjs new file mode 100644 index 00000000..a353f004 --- /dev/null +++ b/packages/treecrdt-sqlite-node/scripts/instrumented-postgres-backend-module.mjs @@ -0,0 +1,15 @@ +import { createPostgresNapiSyncBackendFactory as createBaseFactory } from "@treecrdt/postgres-napi"; + +import { wrapBackendWithProfiler } from "./backend-profiler.mjs"; + +export function createPostgresNapiSyncBackendFactory(url) { + const baseFactory = createBaseFactory(url); + + return { + ...baseFactory, + open: async (docId) => { + const backend = await baseFactory.open(docId); + return wrapBackendWithProfiler(backend, { docId, label: "server-postgres" }); + }, + }; +} diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index fb00ca0f..3df1445c 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -128,6 +128,7 @@ async function main() { pnpm benchmark:sync:direct -- --workloads=sync-balanced-children-payloads-cold-start --counts=10000,50000 --fanout=10 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 --profile-backend TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start Notes: @@ -136,6 +137,7 @@ Notes: - use wss:// for public HTTPS/TLS deployments and ws:// for local/plain HTTP servers - use --fanout=20 to model broader trees; default fanout is 10 - add --first-view to include the immediate local read after sync in the measured duration + - add --profile-backend to capture listOpRefs/getOpsByOpRefs/applyOps timing per backend - extra args are forwarded to packages/treecrdt-sqlite-node/scripts/bench-sync.ts`); return; } From 3c7c21121c7d48d564c30d28264a00da3b453e19 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sat, 21 Mar 2026 14:28:13 +0100 Subject: [PATCH 04/58] feat(sync): shortcut small scoped cold-start sync --- docs/BENCHMARKS.md | 65 ++ packages/sync/protocol/src/sync.ts | 567 +++++----- packages/sync/protocol/tests/smoke.test.ts | 350 +++---- packages/sync/server/postgres-node/src/cli.ts | 109 +- .../sync/server/postgres-node/src/server.ts | 201 ++-- .../scripts/bench-sync.ts | 966 +++++++++++++++++- scripts/run-sync-bench.mjs | 12 +- 7 files changed, 1625 insertions(+), 645 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index edc373a3..e77cb75e 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -75,6 +75,10 @@ pnpm benchmark:sync:remote -- \ Use `--fanout=20` when you want to model a broader notebook tree. +By default, the local sync target runs the Postgres sync server in a spawned child process so local and remote measurements are closer to each other. When you add `--profile-backend`, the local target intentionally switches to the in-process server so per-backend timings are visible inside the benchmark process. + +Local server benchmarks now seed the Postgres backend directly before the timer starts. That keeps the measured path honest, because the actual sync to the client still goes through the real websocket server, while avoiding huge protocol-seed setup costs that are not part of the benchmark question. + ### Time To First Visible Page Add `--first-view` when you want one number that includes: @@ -91,6 +95,26 @@ pnpm benchmark:sync:local -- \ --first-view ``` +For custom `--count` or `--counts` runs, the sync bench now defaults to multiple measured samples instead of silently falling back to one. Use `--iterations=N` and `--warmup=N` when you want explicit control over stability versus runtime. + +### Small-Scope Direct Send + +Add `--direct-send-threshold=N` when you want to experiment with a clean-slate shortcut for small scoped syncs. + +When enabled, if the requesting peer has an empty local result for the requested filter and the responder has at most `N` matching ops, the protocol skips the RIBLT round and sends the scoped ops directly in `opsBatch`. + +This is most relevant for first-view note loading where the client knows the scope root but has not synced its immediate children yet. + +```sh +TREECRDT_POSTGRES_URL=postgres://postgres:postgres@127.0.0.1:55432/postgres \ +pnpm benchmark:sync:local -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --fanout=10 \ + --first-view \ + --direct-send-threshold=64 +``` + ### Backend Call Profiling Add `--profile-backend` when you want per-backend timings for: @@ -110,6 +134,43 @@ pnpm benchmark:sync:local -- \ --profile-backend ``` +### Transport Profiling + +Add `--profile-transport` when you want sync message counts, encoded byte counts, and a short event timeline showing where time is spent across the handshake, RIBLT exchange, and ops batches. + +```sh +TREECRDT_POSTGRES_URL=postgres://postgres:postgres@127.0.0.1:55432/postgres \ +pnpm benchmark:sync:local -- \ + --workloads=sync-balanced-children-cold-start,sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --fanout=10 \ + --profile-transport +``` + +### Hello Stage Profiling + +Add `--profile-hello` when you want the responder-side `hello -> helloAck` path broken into internal stages such as: + +- `maxLamport` +- `listOpRefs` +- `filterOutgoingOps` when auth filtering is active +- decoder setup +- `helloAck` send + +This is the right profiler when the coarse transport timeline says `hello -> helloAck` is expensive and you need to know whether that cost is database work, auth filtering, or protocol setup. + +```sh +TREECRDT_POSTGRES_URL=postgres://postgres:postgres@127.0.0.1:55432/postgres \ +pnpm benchmark:sync:local -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --fanout=10 \ + --first-view \ + --profile-hello +``` + +For `local-postgres-sync-server`, child-process runs capture hello traces by parsing the server process output. For `direct` and in-process debug runs, the benchmark collects the same trace in-process without writing debug noise into the result stream. Remote runs currently only have the coarse transport profile, not internal server hello stages. + ### Local First View Read Path This measures the app-shaped local read immediately after sync: fetch the visible children page plus payloads for the parent and those children. @@ -200,7 +261,11 @@ Common sync flags: - `--targets=direct,local-postgres-sync-server` - `--fanout=10` - `--first-view` +- `--iterations=5` +- `--warmup=1` - `--profile-backend` +- `--profile-transport` +- `--profile-hello` - `--sync-server-url=ws://host/sync` - `--postgres-url=postgres://...` diff --git a/packages/sync/protocol/src/sync.ts b/packages/sync/protocol/src/sync.ts index 24b8590f..fda56a0b 100644 --- a/packages/sync/protocol/src/sync.ts +++ b/packages/sync/protocol/src/sync.ts @@ -1,12 +1,8 @@ -import { RibltDecoder16, RibltEncoder16 } from '@treecrdt/riblt-wasm'; - -import { - AUTH_CAPABILITY_NAME, - isAnyAuthCapability, - isAuthCapability, -} from './auth-capabilities.js'; -import type { SyncAuth, SyncAuthVerifyOpsResult, SyncOpPurpose } from './auth.js'; -import { ErrorCode, RibltFailureReason } from './types.js'; +import { RibltDecoder16, RibltEncoder16 } from "@treecrdt/riblt-wasm"; + +import { AUTH_CAPABILITY_NAME, isAnyAuthCapability, isAuthCapability } from "./auth-capabilities.js"; +import type { SyncAuth, SyncAuthVerifyOpsResult, SyncOpPurpose } from "./auth.js"; +import { ErrorCode, RibltFailureReason } from "./types.js"; import type { Capability, Filter, @@ -22,12 +18,12 @@ import type { SyncBackend, SyncMessage, Unsubscribe, -} from './types.js'; -import type { DuplexTransport } from './transport/index.js'; +} from "./types.js"; +import type { DuplexTransport } from "./transport/index.js"; function randomId(prefix: string): string { const uuid = - typeof globalThis.crypto?.randomUUID === 'function' + typeof globalThis.crypto?.randomUUID === "function" ? globalThis.crypto.randomUUID() : `${Math.random().toString(16).slice(2)}_${Date.now().toString(16)}`; return `${prefix}_${uuid}`; @@ -52,6 +48,10 @@ function deferred(): Pending { return { promise, resolve, reject }; } +const DIRECT_SEND_SMALL_SCOPE_SUPPORT_CAPABILITY = "treecrdt.sync.direct_send_small_scope.v1"; +const DIRECT_SEND_SMALL_SCOPE_REQUEST_CAPABILITY = "treecrdt.sync.direct_send_small_scope.request.v1"; +const DIRECT_SEND_SMALL_SCOPE_FILTER_CAPABILITY = "treecrdt.sync.direct_send_small_scope.filter.v1"; + type ResponderSession = { filter: Filter; round: number; @@ -75,6 +75,7 @@ export type SyncPeerOptions = { maxCodewords?: number; maxOpsPerBatch?: number; maxHelloFilters?: number; + directSendThreshold?: number; requireAuthForFilters?: boolean; auth?: SyncAuth; }; @@ -110,7 +111,7 @@ function sleepUntil(ms: number, signal?: AbortSignal): Promise { const timer = setTimeout(() => { if (done) return; done = true; - signal?.removeEventListener('abort', onAbort); + signal?.removeEventListener("abort", onAbort); resolve(true); }, ms); @@ -118,7 +119,7 @@ function sleepUntil(ms: number, signal?: AbortSignal): Promise { if (done) return; done = true; clearTimeout(timer); - signal?.removeEventListener('abort', onAbort); + signal?.removeEventListener("abort", onAbort); resolve(false); }; @@ -127,20 +128,18 @@ function sleepUntil(ms: number, signal?: AbortSignal): Promise { onAbort(); return; } - signal.addEventListener('abort', onAbort); + signal.addEventListener("abort", onAbort); } }); } const yieldToMacrotask: () => Promise = (() => { - const setImmediateImpl = (globalThis as any).setImmediate as - | undefined - | ((cb: () => void) => void); - if (typeof setImmediateImpl === 'function') { + const setImmediateImpl = (globalThis as any).setImmediate as undefined | ((cb: () => void) => void); + if (typeof setImmediateImpl === "function") { return async () => new Promise((resolve) => setImmediateImpl(resolve)); } - if (typeof MessageChannel !== 'undefined') { + if (typeof MessageChannel !== "undefined") { const queue: Array<() => void> = []; const channel = new MessageChannel(); channel.port1.onmessage = () => { @@ -156,17 +155,65 @@ const yieldToMacrotask: () => Promise = (() => { return async () => new Promise((resolve) => setTimeout(resolve, 0)); })(); +const TRACE_HELLO_ENABLED = + typeof process !== "undefined" && + typeof process?.env?.TREECRDT_SYNC_TRACE_HELLO === "string" && + process.env.TREECRDT_SYNC_TRACE_HELLO !== "0" && + process.env.TREECRDT_SYNC_TRACE_HELLO.toLowerCase() !== "false"; + +type HelloTraceRecord = { + type: "sync-hello-trace"; + docId: string; + stage: string; + ms: number; +} & Record; + +type HelloTraceSink = (record: HelloTraceRecord) => void; + +const HELLO_TRACE_SINK_KEY = "__TREECRDT_SYNC_HELLO_TRACE_SINK__"; + +function getHelloTraceSink(): HelloTraceSink | undefined { + const sink = (globalThis as Record)[HELLO_TRACE_SINK_KEY]; + return typeof sink === "function" ? (sink as HelloTraceSink) : undefined; +} + +function traceHello( + docId: string, + startedAt: number, + stage: string, + extra: Record = {} +): void { + const sink = getHelloTraceSink(); + if (!TRACE_HELLO_ENABLED && !sink) return; + const record: HelloTraceRecord = { + type: "sync-hello-trace", + docId, + stage, + ms: performance.now() - startedAt, + ...extra, + }; + try { + sink?.(record); + } catch { + // debug tracing must never affect sync behavior + } + if (!TRACE_HELLO_ENABLED) return; + try { + console.log(JSON.stringify(record)); + } catch { + // debug tracing must never affect sync behavior + } +} + function bytesToHex(bytes: Uint8Array): string { - let out = ''; - for (const b of bytes) out += b.toString(16).padStart(2, '0'); + let out = ""; + for (const b of bytes) out += b.toString(16).padStart(2, "0"); return out; } function waitForAbort(signal: AbortSignal): Promise { if (signal.aborted) return Promise.resolve(); - return new Promise((resolve) => - signal.addEventListener('abort', () => resolve(), { once: true }), - ); + return new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true })); } type ResponderSubscription = { @@ -185,17 +232,45 @@ function peerAdvertisedOpAuth(capabilities: readonly Capability[]): boolean { return capabilities.some(isAnyAuthCapability); } +function peerSupportsDirectSendSmallScope(capabilities: readonly Capability[]): boolean { + return capabilities.some( + (capability) => + capability.name === DIRECT_SEND_SMALL_SCOPE_SUPPORT_CAPABILITY && + capability.value === "1" + ); +} + +function peerSelectedDirectSendFilter( + capabilities: readonly Capability[], + filterId: string +): boolean { + return capabilities.some( + (capability) => + capability.name === DIRECT_SEND_SMALL_SCOPE_FILTER_CAPABILITY && + capability.value === filterId + ); +} + +function peerRequestedDirectSendFilter( + capabilities: readonly Capability[], + filterId: string +): boolean { + return capabilities.some( + (capability) => + capability.name === DIRECT_SEND_SMALL_SCOPE_REQUEST_CAPABILITY && + capability.value === filterId + ); +} + export class SyncPeer { private readonly maxCodewords: number; private readonly maxOpsPerBatch: number; private readonly maxHelloFilters: number; + private readonly directSendThreshold: number; private readonly requireAuthForFilters: boolean; private readonly auth?: SyncAuth; private readonly transportHasAuth = new WeakMap>, boolean>(); - private readonly transportPeerCapabilities = new WeakMap< - DuplexTransport>, - Hello['capabilities'] - >(); + private readonly transportPeerCapabilities = new WeakMap>, Hello["capabilities"]>(); private readonly responderSessions = new Map>(); private readonly initiatorSessions = new Map>(); private readonly responderSubscriptions = new Map>(); @@ -208,18 +283,22 @@ export class SyncPeer { constructor( private readonly backend: SyncBackend, - opts: SyncPeerOptions = {}, + opts: SyncPeerOptions = {} ) { this.maxCodewords = opts.maxCodewords ?? 50_000; this.maxOpsPerBatch = opts.maxOpsPerBatch ?? 5_000; this.auth = opts.auth; this.maxHelloFilters = opts.maxHelloFilters ?? 8; + this.directSendThreshold = opts.directSendThreshold ?? 0; + if (!Number.isInteger(this.directSendThreshold) || this.directSendThreshold < 0) { + throw new Error(`invalid directSendThreshold: ${opts.directSendThreshold}`); + } this.requireAuthForFilters = opts.requireAuthForFilters ?? Boolean(opts.auth); } attach( transport: DuplexTransport>, - opts: SyncPeerAttachOptions = {}, + opts: SyncPeerAttachOptions = {} ): () => void { return transport.onMessage((msg) => { void this.handleMessage(transport, msg).catch((error) => { @@ -291,14 +370,12 @@ export class SyncPeer { if (this.auth?.filterOutgoingOps && ops.length > 0) { const allowed = await this.auth.filterOutgoingOps(ops, { docId: this.backend.docId, - purpose: 'subscribe', + purpose: "subscribe", filter: sub.filter, capabilities: peerCaps, }); if (allowed.length !== ops.length) { - throw new Error( - `filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`, - ); + throw new Error(`filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`); } const allowedRefs: OpRef[] = []; @@ -324,24 +401,15 @@ export class SyncPeer { } const shouldAttachAuth = peerAdvertisedOpAuth(peerCaps); - const auth = - shouldAttachAuth && this.auth?.signOps - ? await this.auth.signOps(ops, { - docId: this.backend.docId, - purpose: 'subscribe', - filterId: sub.subscriptionId, - }) - : undefined; - if (auth && auth.length !== ops.length) - throw new Error(`signOps returned ${auth.length} entries for ${ops.length} ops`); + const auth = shouldAttachAuth && this.auth?.signOps + ? await this.auth.signOps(ops, { docId: this.backend.docId, purpose: "subscribe", filterId: sub.subscriptionId }) + : undefined; + if (auth && auth.length !== ops.length) throw new Error(`signOps returned ${auth.length} entries for ${ops.length} ops`); const done = start + this.maxOpsPerBatch >= newOpRefs.length; await sub.transport.send({ v: 0, docId: this.backend.docId, - payload: { - case: 'opsBatch', - value: { filterId: sub.subscriptionId, ops, ...(auth ? { auth } : {}), done }, - }, + payload: { case: "opsBatch", value: { filterId: sub.subscriptionId, ops, ...(auth ? { auth } : {}), done } }, }); for (const r of chunk) sub.sentOpRefs.add(bytesToHex(r)); @@ -352,13 +420,28 @@ export class SyncPeer { async syncOnce( transport: DuplexTransport>, filter: Filter, - opts: SyncOnceOptions = {}, + opts: SyncOnceOptions = {} ): Promise { - const filterId = randomId('f'); + const filterId = randomId("f"); const round = 0; const maxLamport = await this.backend.maxLamport(); - const capabilities = - (await this.auth?.helloCapabilities?.({ docId: this.backend.docId })) ?? []; + const localOpRefsBeforeHello = await this.backend.listOpRefs(filter); + const capabilities = (await this.auth?.helloCapabilities?.({ docId: this.backend.docId })) ?? []; + if (!peerSupportsDirectSendSmallScope(capabilities)) { + capabilities.push({ + name: DIRECT_SEND_SMALL_SCOPE_SUPPORT_CAPABILITY, + value: "1", + }); + } + if ( + localOpRefsBeforeHello.length === 0 && + !peerRequestedDirectSendFilter(capabilities, filterId) + ) { + capabilities.push({ + name: DIRECT_SEND_SMALL_SCOPE_REQUEST_CAPABILITY, + value: filterId, + }); + } const hello: Hello = { capabilities, filters: [{ id: filterId, filter }], maxLamport }; const session: InitiatorSession = { @@ -378,9 +461,17 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'hello', value: hello }, + payload: { case: "hello", value: hello }, }); - await session.ack.promise; + const ack = await session.ack.promise; + + if ( + localOpRefsBeforeHello.length === 0 && + peerSelectedDirectSendFilter(ack.capabilities, filterId) + ) { + await session.receivedOps.promise; + return; + } let opRefs = await this.backend.listOpRefs(filter); @@ -391,14 +482,12 @@ export class SyncPeer { const ops = await this.backend.getOpsByOpRefs(opRefs); const allowed = await this.auth.filterOutgoingOps(ops, { docId: this.backend.docId, - purpose: 'reconcile', + purpose: "reconcile", filter, capabilities: peerCaps, }); if (allowed.length !== ops.length) { - throw new Error( - `filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`, - ); + throw new Error(`filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`); } opRefs = opRefs.filter((_r, idx) => allowed[idx] === true); } @@ -414,10 +503,7 @@ export class SyncPeer { if (session.codewordCredits <= 0) { const wakeForCredits = session.codewordCreditSignal.promise; await Promise.race([ - session.terminalStatus.promise.then( - () => undefined, - () => undefined, - ), + session.terminalStatus.promise.then(() => undefined, () => undefined), wakeForCredits, ]); continue; @@ -425,7 +511,7 @@ export class SyncPeer { session.codewordCredits -= 1; const startIndex = nextIndex; - const codewords: RibltCodewords['codewords'] = []; + const codewords: RibltCodewords["codewords"] = []; for (let i = 0; i < codewordsPerMessage && nextIndex < maxCodewords; i += 1) { codewords.push(enc.nextCodeword() as any); nextIndex += 1n; @@ -435,25 +521,25 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'ribltCodewords', + case: "ribltCodewords", value: { filterId, round, startIndex, codewords }, }, }); await yieldToMacrotask(); } - if (!session.done) throw new Error('riblt: max codewords exceeded'); + if (!session.done) throw new Error("riblt: max codewords exceeded"); const status = await session.terminalStatus.promise; - if (status.payload.case === 'failed') { + if (status.payload.case === "failed") { const { reason, message } = status.payload.value; const name = RibltFailureReason[reason] ?? String(reason); - const detail = message ? `: ${message}` : ''; + const detail = message ? `: ${message}` : ""; throw new Error(`riblt: ${name}${detail}`); } const receiverMissing = - status.payload.case === 'decoded' ? status.payload.value.receiverMissing : []; + status.payload.case === "decoded" ? status.payload.value.receiverMissing : []; if (receiverMissing.length > 0) { await this.sendOpsBatches(transport, filterId, receiverMissing, { maxOpsPerBatch: opts.maxOpsPerBatch, @@ -462,7 +548,7 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'opsBatch', value: { filterId, ops: [], done: true } }, + payload: { case: "opsBatch", value: { filterId, ops: [], done: true } }, }); } @@ -476,7 +562,7 @@ export class SyncPeer { transport: DuplexTransport>, filterId: string, opRefs: OpRef[], - opts: { maxOpsPerBatch?: number } = {}, + opts: { maxOpsPerBatch?: number; filter?: Filter } = {} ): Promise { const maxOpsPerBatch = opts.maxOpsPerBatch ?? this.maxOpsPerBatch; if (!Number.isFinite(maxOpsPerBatch) || maxOpsPerBatch <= 0) { @@ -484,6 +570,7 @@ export class SyncPeer { } const filter = + opts.filter ?? this.responderSessions.get(filterId)?.filter ?? this.initiatorSessions.get(filterId)?.filter ?? undefined; @@ -493,7 +580,7 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'opsBatch', value: { filterId, ops: [], done: true } }, + payload: { case: "opsBatch", value: { filterId, ops: [], done: true } }, }); return; } @@ -505,14 +592,12 @@ export class SyncPeer { if (filter && this.auth?.filterOutgoingOps && ops.length > 0) { const allowed = await this.auth.filterOutgoingOps(ops, { docId: this.backend.docId, - purpose: 'reconcile', + purpose: "reconcile", filter, capabilities: peerCaps, }); if (allowed.length !== ops.length) { - throw new Error( - `filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`, - ); + throw new Error(`filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`); } const nextOps: Op[] = []; @@ -529,21 +614,16 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'opsBatch', value: { filterId, ops: [], done: true } }, + payload: { case: "opsBatch", value: { filterId, ops: [], done: true } }, }); } continue; } const shouldAttachAuth = peerAdvertisedOpAuth(peerCaps); - const auth = - shouldAttachAuth && this.auth?.signOps - ? await this.auth.signOps(ops, { - docId: this.backend.docId, - purpose: 'reconcile', - filterId, - }) - : undefined; + const auth = shouldAttachAuth && this.auth?.signOps + ? await this.auth.signOps(ops, { docId: this.backend.docId, purpose: "reconcile", filterId }) + : undefined; if (auth && auth.length !== ops.length) { throw new Error(`signOps returned ${auth.length} entries for ${ops.length} ops`); } @@ -551,7 +631,7 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'opsBatch', value: { filterId, ops, ...(auth ? { auth } : {}), done } }, + payload: { case: "opsBatch", value: { filterId, ops, ...(auth ? { auth } : {}), done } }, }); await yieldToMacrotask(); } @@ -560,14 +640,14 @@ export class SyncPeer { subscribe( transport: DuplexTransport>, filter: Filter, - opts: SyncSubscribeOptions = {}, + opts: SyncSubscribeOptions = {} ): SyncSubscription { const controller = new AbortController(); if (opts.signal) { if (opts.signal.aborted) { controller.abort(); } else { - opts.signal.addEventListener('abort', () => controller.abort(), { once: true }); + opts.signal.addEventListener("abort", () => controller.abort(), { once: true }); } } @@ -578,11 +658,8 @@ export class SyncPeer { const maxCodewords = opts.maxCodewords; const maxOpsPerBatch = opts.maxOpsPerBatch; - const subscriptionId = randomId('sub'); - const session: InitiatorSubscription = { - ack: deferred(), - failed: deferred(), - }; + const subscriptionId = randomId("sub"); + const session: InitiatorSubscription = { ack: deferred(), failed: deferred() }; this.initiatorSubscriptions.set(subscriptionId, session); const ready = deferred(); let readySettled = false; @@ -615,27 +692,27 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'hello', value: { capabilities, filters: [], maxLamport } }, + payload: { case: "hello", value: { capabilities, filters: [], maxLamport } }, }); } await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'subscribe', value: { subscriptionId, filter } }, + payload: { case: "subscribe", value: { subscriptionId, filter } }, }); sentSubscribe = true; const ackOrAbort = await Promise.race([ - session.ack.promise.then(() => ({ case: 'ack' as const })), - session.failed.promise.then((err) => ({ case: 'failed' as const, err })), - waitForAbort(signal).then(() => ({ case: 'aborted' as const })), + session.ack.promise.then(() => ({ case: "ack" as const })), + session.failed.promise.then((err) => ({ case: "failed" as const, err })), + waitForAbort(signal).then(() => ({ case: "aborted" as const })), ]); - if (ackOrAbort.case === 'aborted') { + if (ackOrAbort.case === "aborted") { resolveReady(); return; } - if (ackOrAbort.case === 'failed') { + if (ackOrAbort.case === "failed") { rejectReady(ackOrAbort.err); throw ackOrAbort.err; } @@ -645,11 +722,7 @@ export class SyncPeer { return; } if (immediate) { - await this.syncOnce(transport, filter, { - codewordsPerMessage, - maxCodewords, - maxOpsPerBatch, - }); + await this.syncOnce(transport, filter, { codewordsPerMessage, maxCodewords, maxOpsPerBatch }); } resolveReady(); @@ -658,11 +731,7 @@ export class SyncPeer { const slept = await sleepUntil(intervalMs, signal); if (!slept) break; if (signal.aborted) break; - await this.syncOnce(transport, filter, { - codewordsPerMessage, - maxCodewords, - maxOpsPerBatch, - }); + await this.syncOnce(transport, filter, { codewordsPerMessage, maxCodewords, maxOpsPerBatch }); } } else { await Promise.race([ @@ -683,7 +752,7 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'unsubscribe', value: { subscriptionId } }, + payload: { case: "unsubscribe", value: { subscriptionId } }, }); } catch { // ignore transport failures during teardown @@ -697,11 +766,11 @@ export class SyncPeer { private async handleMessage( transport: DuplexTransport>, - msg: SyncMessage, + msg: SyncMessage ): Promise { if (msg.docId !== this.backend.docId) return; - if (msg.payload.case === 'error') { + if (msg.payload.case === "error") { try { await this.onError(msg.payload.value); } catch { @@ -712,28 +781,28 @@ export class SyncPeer { try { switch (msg.payload.case) { - case 'hello': + case "hello": await this.onHello(transport, msg.payload.value); return; - case 'helloAck': + case "helloAck": await this.onHelloAck(transport, msg.payload.value); return; - case 'ribltCodewords': + case "ribltCodewords": await this.onRibltCodewords(transport, msg.payload.value); return; - case 'ribltStatus': + case "ribltStatus": await this.onRibltStatus(msg.payload.value); return; - case 'opsBatch': + case "opsBatch": await this.onOpsBatch(msg.payload.value); return; - case 'subscribe': + case "subscribe": await this.onSubscribe(transport, msg.payload.value); return; - case 'subscribeAck': + case "subscribeAck": await this.onSubscribeAck(msg.payload.value); return; - case 'unsubscribe': + case "unsubscribe": await this.onUnsubscribe(msg.payload.value); return; default: { @@ -745,17 +814,17 @@ export class SyncPeer { let filterId: string | undefined; let subscriptionId: string | undefined; switch (msg.payload.case) { - case 'ribltCodewords': - case 'ribltStatus': - case 'opsBatch': + case "ribltCodewords": + case "ribltStatus": + case "opsBatch": filterId = msg.payload.value.filterId; - if (msg.payload.case === 'opsBatch' && this.initiatorSubscriptions.has(filterId)) { + if (msg.payload.case === "opsBatch" && this.initiatorSubscriptions.has(filterId)) { subscriptionId = filterId; } break; - case 'subscribe': - case 'subscribeAck': - case 'unsubscribe': + case "subscribe": + case "subscribeAck": + case "unsubscribe": subscriptionId = msg.payload.value.subscriptionId; break; } @@ -763,7 +832,7 @@ export class SyncPeer { try { await this.onError({ code: ErrorCode.ERROR_CODE_UNSPECIFIED, - message: String(err?.message ?? err ?? 'error'), + message: String(err?.message ?? err ?? "error"), ...(filterId ? { filterId } : {}), ...(subscriptionId ? { subscriptionId } : {}), }); @@ -772,10 +841,10 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'error', + case: "error", value: { code: ErrorCode.ERROR_CODE_UNSPECIFIED, - message: String(err?.message ?? err ?? 'error'), + message: String(err?.message ?? err ?? "error"), ...(filterId ? { filterId } : {}), ...(subscriptionId ? { subscriptionId } : {}), }, @@ -787,36 +856,54 @@ export class SyncPeer { } } - private async onHello(transport: DuplexTransport>, hello: Hello): Promise { + private async onHello( + transport: DuplexTransport>, + hello: Hello + ): Promise { + const traceStartedAt = performance.now(); + traceHello(this.backend.docId, traceStartedAt, "start", { + filters: hello.filters.length, + capabilities: hello.capabilities.length, + }); const hasAuthCapability = hello.capabilities.some(isAuthCapability); + const supportsDirectSendSmallScope = peerSupportsDirectSendSmallScope( + hello.capabilities + ); // Record the presence of auth capabilities immediately so concurrent messages (e.g. Subscribe) // can't race and get rejected before `onHello` completes. if (hasAuthCapability) this.transportHasAuth.set(transport, true); - if (peerAdvertisedOpAuth(hello.capabilities)) - this.transportPeerCapabilities.set(transport, hello.capabilities); + if (peerAdvertisedOpAuth(hello.capabilities)) this.transportPeerCapabilities.set(transport, hello.capabilities); - let ackCapabilities: HelloAck['capabilities'] = []; + let ackCapabilities: HelloAck["capabilities"] = []; try { ackCapabilities = (await this.auth?.onHello?.(hello, { docId: this.backend.docId })) ?? []; + traceHello(this.backend.docId, traceStartedAt, "after-auth-onHello", { + ackCapabilities: ackCapabilities.length, + }); } catch (err: any) { await transport.send({ v: 0, docId: this.backend.docId, payload: { - case: 'error', - value: { - code: ErrorCode.ERROR_CODE_UNSPECIFIED, - message: String(err?.message ?? err ?? 'auth error'), - }, + case: "error", + value: { code: ErrorCode.ERROR_CODE_UNSPECIFIED, message: String(err?.message ?? err ?? "auth error") }, }, }); return; } const maxLamport = await this.backend.maxLamport(); + traceHello(this.backend.docId, traceStartedAt, "after-maxLamport", { + maxLamport: Number(maxLamport), + }); const acceptedFilters: string[] = []; - const rejectedFilters: HelloAck['rejectedFilters'] = []; + const rejectedFilters: HelloAck["rejectedFilters"] = []; + const directSendFilters: Array<{ + id: string; + filter: Filter; + opRefs: OpRef[]; + }> = []; for (let i = 0; i < hello.filters.length; i += 1) { const spec = hello.filters[i]!; @@ -837,22 +924,18 @@ export class SyncPeer { rejectedFilters.push({ id, reason: ErrorCode.UNAUTHORIZED, - message: `missing "${AUTH_CAPABILITY_NAME}" token; send a valid capability token in Hello.capabilities`, + message: `missing "${AUTH_CAPABILITY_NAME}" token; send a valid capability token in Hello.capabilities`, }); continue; } try { - await this.auth?.authorizeFilter?.(filter, { - docId: this.backend.docId, - purpose: 'hello', - capabilities: hello.capabilities, - }); + await this.auth?.authorizeFilter?.(filter, { docId: this.backend.docId, purpose: "hello", capabilities: hello.capabilities }); } catch (err: any) { rejectedFilters.push({ id, reason: ErrorCode.UNAUTHORIZED, - message: String(err?.message ?? err ?? 'unauthorized filter'), + message: String(err?.message ?? err ?? "unauthorized filter"), }); continue; } @@ -860,43 +943,77 @@ export class SyncPeer { let localOpRefs: OpRef[]; try { localOpRefs = await this.backend.listOpRefs(filter); + traceHello(this.backend.docId, traceStartedAt, "after-listOpRefs", { + filterId: id, + opRefs: localOpRefs.length, + }); } catch (err: any) { rejectedFilters.push({ id, reason: ErrorCode.FILTER_NOT_SUPPORTED, - message: String(err?.message ?? err ?? 'filter not supported'), + message: String(err?.message ?? err ?? "filter not supported"), }); continue; } - if (!('all' in filter) && this.auth?.filterOutgoingOps && localOpRefs.length > 0) { + if (!("all" in filter) && this.auth?.filterOutgoingOps && localOpRefs.length > 0) { try { const ops = await this.backend.getOpsByOpRefs(localOpRefs); const allowed = await this.auth.filterOutgoingOps(ops, { docId: this.backend.docId, - purpose: 'hello', + purpose: "hello", filter, capabilities: hello.capabilities, }); if (allowed.length !== ops.length) { - throw new Error( - `filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`, - ); + throw new Error(`filterOutgoingOps returned ${allowed.length} flags for ${ops.length} ops`); } localOpRefs = localOpRefs.filter((_r, idx) => allowed[idx] === true); + traceHello(this.backend.docId, traceStartedAt, "after-filterOutgoingOps", { + filterId: id, + fetchedOps: ops.length, + allowedOpRefs: localOpRefs.length, + }); } catch (err: any) { rejectedFilters.push({ id, reason: ErrorCode.UNAUTHORIZED, - message: String(err?.message ?? err ?? 'failed to filter ops'), + message: String(err?.message ?? err ?? "failed to filter ops"), }); continue; } } acceptedFilters.push(id); + + if ( + supportsDirectSendSmallScope && + this.directSendThreshold > 0 && + peerRequestedDirectSendFilter(hello.capabilities, id) && + localOpRefs.length <= this.directSendThreshold + ) { + ackCapabilities.push({ + name: DIRECT_SEND_SMALL_SCOPE_FILTER_CAPABILITY, + value: id, + }); + directSendFilters.push({ + id, + filter, + opRefs: localOpRefs, + }); + traceHello(this.backend.docId, traceStartedAt, "after-direct-send-selection", { + filterId: id, + opRefs: localOpRefs.length, + }); + continue; + } + const decoder = new RibltDecoder16(); for (const r of localOpRefs) decoder.addLocalSymbol(r); + traceHello(this.backend.docId, traceStartedAt, "after-decoder-setup", { + filterId: id, + opRefs: localOpRefs.length, + }); this.responderSessions.set(id, { filter, round: 0, @@ -909,7 +1026,7 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'helloAck', + case: "helloAck", value: { capabilities: ackCapabilities, acceptedFilters, @@ -918,18 +1035,24 @@ export class SyncPeer { }, }, }); + traceHello(this.backend.docId, traceStartedAt, "after-helloAck-send", { + acceptedFilters: acceptedFilters.length, + rejectedFilters: rejectedFilters.length, + }); + + for (const directSend of directSendFilters) { + await this.sendOpsBatches(transport, directSend.id, directSend.opRefs, { + filter: directSend.filter, + }); + } } - private async onHelloAck( - transport: DuplexTransport>, - ack: HelloAck, - ): Promise { + private async onHelloAck(transport: DuplexTransport>, ack: HelloAck): Promise { await this.auth?.onHelloAck?.(ack, { docId: this.backend.docId }); const hasAuthCapability = ack.capabilities.some(isAuthCapability); if (hasAuthCapability) this.transportHasAuth.set(transport, true); - if (peerAdvertisedOpAuth(ack.capabilities)) - this.transportPeerCapabilities.set(transport, ack.capabilities); + if (peerAdvertisedOpAuth(ack.capabilities)) this.transportPeerCapabilities.set(transport, ack.capabilities); for (const id of ack.acceptedFilters) { const session = this.initiatorSessions.get(id); @@ -939,7 +1062,7 @@ export class SyncPeer { const session = this.initiatorSessions.get(rej.id); if (session) { const reason = ErrorCode[rej.reason] ?? String(rej.reason); - const detail = rej.message ? `: ${rej.message}` : ''; + const detail = rej.message ? `: ${rej.message}` : ""; session.ack.reject(new Error(`${reason}${detail}`)); } } @@ -947,7 +1070,7 @@ export class SyncPeer { private async onRibltCodewords( transport: DuplexTransport>, - msg: RibltCodewords, + msg: RibltCodewords ): Promise { const session = this.responderSessions.get(msg.filterId); if (!session) return; @@ -958,11 +1081,11 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'ribltStatus', + case: "ribltStatus", value: { filterId: msg.filterId, round: msg.round, - payload: { case: 'failed', value: { reason: RibltFailureReason.OUT_OF_ORDER } }, + payload: { case: "failed", value: { reason: RibltFailureReason.OUT_OF_ORDER } }, }, }, }); @@ -982,15 +1105,15 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'ribltStatus', + case: "ribltStatus", value: { filterId: msg.filterId, round: msg.round, payload: { - case: 'failed', + case: "failed", value: { reason: RibltFailureReason.DECODE_FAILED, - message: String(err?.message ?? err ?? ''), + message: String(err?.message ?? err ?? ""), }, }, }, @@ -1006,29 +1129,27 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'ribltStatus', + case: "ribltStatus", value: { filterId: msg.filterId, round: msg.round, - payload: { - case: 'failed', - value: { reason: RibltFailureReason.MAX_CODEWORDS_EXCEEDED }, - }, + payload: { case: "failed", value: { reason: RibltFailureReason.MAX_CODEWORDS_EXCEEDED } }, }, }, }); this.responderSessions.delete(msg.filterId); - } else { + } + else { await transport.send({ v: 0, docId: this.backend.docId, payload: { - case: 'ribltStatus', + case: "ribltStatus", value: { filterId: msg.filterId, round: msg.round, payload: { - case: 'more', + case: "more", value: { codewordsReceived: session.decoder.codewordsReceived(), credits: 1 }, }, }, @@ -1046,12 +1167,12 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'ribltStatus', + case: "ribltStatus", value: { filterId: msg.filterId, round: msg.round, payload: { - case: 'decoded', + case: "decoded", value: { senderMissing, receiverMissing, codewordsReceived }, }, }, @@ -1064,7 +1185,7 @@ export class SyncPeer { await transport.send({ v: 0, docId: this.backend.docId, - payload: { case: 'opsBatch', value: { filterId: msg.filterId, ops: [], done: true } }, + payload: { case: "opsBatch", value: { filterId: msg.filterId, ops: [], done: true } }, }); } @@ -1073,7 +1194,7 @@ export class SyncPeer { private async onSubscribe( transport: DuplexTransport>, - msg: Subscribe, + msg: Subscribe ): Promise { if (!msg.subscriptionId) return; if (!msg.filter) return; @@ -1083,7 +1204,7 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'error', + case: "error", value: { code: ErrorCode.UNAUTHORIZED, message: `missing "${AUTH_CAPABILITY_NAME}" token; send Hello before Subscribe`, @@ -1096,20 +1217,16 @@ export class SyncPeer { try { const peerCaps = this.transportPeerCapabilities.get(transport) ?? []; - await this.auth?.authorizeFilter?.(msg.filter, { - docId: this.backend.docId, - purpose: 'subscribe', - capabilities: peerCaps, - }); + await this.auth?.authorizeFilter?.(msg.filter, { docId: this.backend.docId, purpose: "subscribe", capabilities: peerCaps }); } catch (err: any) { await transport.send({ v: 0, docId: this.backend.docId, payload: { - case: 'error', + case: "error", value: { code: ErrorCode.UNAUTHORIZED, - message: String(err?.message ?? err ?? 'unauthorized filter'), + message: String(err?.message ?? err ?? "unauthorized filter"), subscriptionId: msg.subscriptionId, }, }, @@ -1137,7 +1254,7 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'subscribeAck', + case: "subscribeAck", value: { subscriptionId: msg.subscriptionId, currentLamport: maxLamport }, }, }); @@ -1151,10 +1268,10 @@ export class SyncPeer { v: 0, docId: this.backend.docId, payload: { - case: 'error', + case: "error", value: { code: ErrorCode.FILTER_NOT_SUPPORTED, - message: String(err?.message ?? err ?? 'subscribe failed'), + message: String(err?.message ?? err ?? "subscribe failed"), subscriptionId: msg.subscriptionId, }, }, @@ -1172,12 +1289,7 @@ export class SyncPeer { this.responderSubscriptions.delete(msg.subscriptionId); } - private async onError(err: { - code: ErrorCode; - message: string; - filterId?: string; - subscriptionId?: string; - }): Promise { + private async onError(err: { code: ErrorCode; message: string; filterId?: string; subscriptionId?: string }): Promise { if (err.subscriptionId) { const sub = this.initiatorSubscriptions.get(err.subscriptionId); if (sub) { @@ -1240,7 +1352,7 @@ export class SyncPeer { if (!session) return; if (status.round !== session.round) return; if (session.done) return; - if (status.payload.case === 'more') { + if (status.payload.case === "more") { const credits = Math.max(1, Math.trunc(status.payload.value.credits)); session.codewordCredits += credits; const signal = session.codewordCreditSignal; @@ -1256,14 +1368,10 @@ export class SyncPeer { } private async onOpsBatch(batch: OpsBatch): Promise { - const purpose: SyncOpPurpose = this.initiatorSubscriptions.has(batch.filterId) - ? 'subscribe' - : 'reconcile'; + const purpose: SyncOpPurpose = this.initiatorSubscriptions.has(batch.filterId) ? "subscribe" : "reconcile"; const auth = batch.auth; if (auth && auth.length !== batch.ops.length) { - throw new Error( - `OpsBatch.auth length ${auth.length} does not match ops length ${batch.ops.length}`, - ); + throw new Error(`OpsBatch.auth length ${auth.length} does not match ops length ${batch.ops.length}`); } const verifyRes = await this.auth?.verifyOps?.(batch.ops, auth, { @@ -1274,21 +1382,15 @@ export class SyncPeer { const dispositions = verifyRes === undefined ? undefined - : ((verifyRes as SyncAuthVerifyOpsResult)?.dispositions ?? + : (verifyRes as SyncAuthVerifyOpsResult)?.dispositions ?? (() => { - throw new Error('verifyOps must return void or { dispositions: [...] }'); - })()); + throw new Error("verifyOps must return void or { dispositions: [...] }"); + })(); if (dispositions && dispositions.length !== batch.ops.length) { - throw new Error( - `verifyOps returned ${dispositions.length} dispositions for ${batch.ops.length} ops`, - ); + throw new Error(`verifyOps returned ${dispositions.length} dispositions for ${batch.ops.length} ops`); } if (auth && auth.length > 0) { - await this.auth?.onVerifiedOps?.(batch.ops, auth, { - docId: this.backend.docId, - purpose, - filterId: batch.filterId, - }); + await this.auth?.onVerifiedOps?.(batch.ops, auth, { docId: this.backend.docId, purpose, filterId: batch.filterId }); } const pending: PendingOp[] = []; @@ -1297,29 +1399,22 @@ export class SyncPeer { for (let i = 0; i < batch.ops.length; i += 1) { const op = batch.ops[i]!; const d = dispositions?.[i]; - if (!d || d.status === 'allow') { + if (!d || d.status === "allow") { allowedOps.push(op); continue; } - if (d.status !== 'pending_context') { + if (d.status !== "pending_context") { throw new Error(`unknown disposition: ${(d as any)?.status ?? String(d)}`); } if (!auth) { - throw new Error('verifyOps returned pending_context but OpsBatch.auth is missing'); + throw new Error("verifyOps returned pending_context but OpsBatch.auth is missing"); } - pending.push({ - op, - auth: auth[i]!, - reason: 'missing_context', - ...(d.message ? { message: d.message } : {}), - }); + pending.push({ op, auth: auth[i]!, reason: "missing_context", ...(d.message ? { message: d.message } : {}) }); } if (pending.length > 0) { if (!this.backend.storePendingOps) { - throw new Error( - 'received ops requiring pending-context handling, but backend.storePendingOps is not implemented', - ); + throw new Error("received ops requiring pending-context handling, but backend.storePendingOps is not implemented"); } await this.backend.storePendingOps(pending); } @@ -1354,11 +1449,7 @@ export class SyncPeer { let appliedAny = false; for (const p of pending) { - const ctx = { - docId: this.backend.docId, - purpose: 'reprocess_pending' as const, - filterId: '__pending__', - }; + const ctx = { docId: this.backend.docId, purpose: "reprocess_pending" as const, filterId: "__pending__" }; let res: void | SyncAuthVerifyOpsResult; try { res = await this.auth!.verifyOps!([p.op], [p.auth], ctx); @@ -1373,12 +1464,12 @@ export class SyncPeer { const dispositions = res === undefined ? undefined - : ((res as SyncAuthVerifyOpsResult)?.dispositions ?? + : (res as SyncAuthVerifyOpsResult)?.dispositions ?? (() => { - throw new Error('verifyOps must return void or { dispositions: [...] }'); - })()); + throw new Error("verifyOps must return void or { dispositions: [...] }"); + })(); const d = dispositions?.[0]; - if (d && d.status === 'pending_context') continue; + if (d && d.status === "pending_context") continue; await this.backend.applyOps([p.op]); await this.backend.deletePendingOps!([p.op]); @@ -1389,7 +1480,7 @@ export class SyncPeer { if (appliedAny) void this.notifyLocalUpdate(); if (!progress) return; } - throw new Error('pending-op reprocessing exceeded max rounds'); + throw new Error("pending-op reprocessing exceeded max rounds"); })().finally(() => { this.reprocessPendingRunning = false; }); diff --git a/packages/sync/protocol/tests/smoke.test.ts b/packages/sync/protocol/tests/smoke.test.ts index 609d97ca..e121e74f 100644 --- a/packages/sync/protocol/tests/smoke.test.ts +++ b/packages/sync/protocol/tests/smoke.test.ts @@ -1,21 +1,21 @@ -import { createHash } from 'node:crypto'; +import { createHash } from "node:crypto"; -import { expect, test } from 'vitest'; +import { expect, test } from "vitest"; -import type { Operation } from '@treecrdt/interface'; -import { bytesToHex, nodeIdToBytes16 } from '@treecrdt/interface/ids'; -import { makeOp, nodeIdFromInt } from '@treecrdt/benchmark'; +import type { Operation } from "@treecrdt/interface"; +import { bytesToHex, nodeIdToBytes16 } from "@treecrdt/interface/ids"; +import { makeOp, nodeIdFromInt } from "@treecrdt/benchmark"; -import { treecrdtSyncV0ProtobufCodec } from '../dist/protobuf.js'; -import { SyncPeer } from '../dist/sync.js'; -import { createInMemoryConnectedPeers } from '../dist/in-memory.js'; -import { wrapDuplexTransportWithCodec } from '../dist/transport/index.js'; -import type { DuplexTransport } from '../dist/transport/index.js'; -import type { Filter, OpRef, SyncBackend, SyncMessage } from '../dist/types.js'; +import { treecrdtSyncV0ProtobufCodec } from "../dist/protobuf.js"; +import { SyncPeer } from "../dist/sync.js"; +import { createInMemoryConnectedPeers } from "../dist/in-memory.js"; +import { wrapDuplexTransportWithCodec } from "../dist/transport/index.js"; +import type { DuplexTransport } from "../dist/transport/index.js"; +import type { Filter, OpRef, SyncBackend, SyncMessage } from "../dist/types.js"; function opRefFor(docId: string, replica: string, counter: number): OpRef { - const h = createHash('sha256'); - h.update('treecrdt/opref/test'); + const h = createHash("sha256"); + h.update("treecrdt/opref/test"); h.update(docId); h.update(replica); h.update(String(counter)); @@ -41,16 +41,16 @@ function orderKeyFromPosition(position: number): Uint8Array { function replicaFromLabel(label: string): Uint8Array { const encoded = new TextEncoder().encode(label); - if (encoded.length === 0) throw new Error('replica label must not be empty'); + if (encoded.length === 0) throw new Error("replica label must not be empty"); const out = new Uint8Array(32); for (let i = 0; i < out.length; i += 1) out[i] = encoded[i % encoded.length]!; return out; } const replicas = { - a: replicaFromLabel('a'), - b: replicaFromLabel('b'), - s: replicaFromLabel('s'), + a: replicaFromLabel("a"), + b: replicaFromLabel("b"), + s: replicaFromLabel("s"), }; const replicaHex = { @@ -59,9 +59,7 @@ const replicaHex = { s: bytesToHex(replicas.s), }; -function createTimedDuplex( - opts: { aToBDelayMs?: number; bToADelayMs?: number } = {}, -): [DuplexTransport, DuplexTransport] { +function createTimedDuplex(opts: { aToBDelayMs?: number; bToADelayMs?: number } = {}): [DuplexTransport, DuplexTransport] { const aHandlers = new Set<(msg: M) => void>(); const bHandlers = new Set<(msg: M) => void>(); const aToBDelayMs = opts.aToBDelayMs ?? 0; @@ -95,17 +93,17 @@ function createTimedDuplex( } function createLoggedTimedDuplex( - opts: { aToBDelayMs?: number; bToADelayMs?: number } = {}, -): [DuplexTransport, DuplexTransport, Array<{ dir: 'aToB' | 'bToA'; msg: M }>] { + opts: { aToBDelayMs?: number; bToADelayMs?: number } = {} +): [DuplexTransport, DuplexTransport, Array<{ dir: "aToB" | "bToA"; msg: M }>] { const aHandlers = new Set<(msg: M) => void>(); const bHandlers = new Set<(msg: M) => void>(); - const log: Array<{ dir: 'aToB' | 'bToA'; msg: M }> = []; + const log: Array<{ dir: "aToB" | "bToA"; msg: M }> = []; const aToBDelayMs = opts.aToBDelayMs ?? 0; const bToADelayMs = opts.bToADelayMs ?? 0; const a: DuplexTransport = { async send(msg) { - log.push({ dir: 'aToB', msg }); + log.push({ dir: "aToB", msg }); setTimeout(() => { for (const h of bHandlers) h(msg); }, aToBDelayMs); @@ -118,7 +116,7 @@ function createLoggedTimedDuplex( const b: DuplexTransport = { async send(msg) { - log.push({ dir: 'bToA', msg }); + log.push({ dir: "bToA", msg }); setTimeout(() => { for (const h of aHandlers) h(msg); }, bToADelayMs); @@ -146,7 +144,7 @@ function createPeers(a: SyncBackend, b: SyncBackend) { async function waitUntil( predicate: () => Promise | boolean, - opts: { timeoutMs?: number; intervalMs?: number; message?: string } = {}, + opts: { timeoutMs?: number; intervalMs?: number; message?: string } = {} ): Promise { const timeoutMs = opts.timeoutMs ?? 2_000; const intervalMs = opts.intervalMs ?? 10; @@ -162,7 +160,7 @@ async function waitUntil( } } -const TRASH_HEX = 'ff'.repeat(16); +const TRASH_HEX = "ff".repeat(16); class MemoryBackend implements SyncBackend { readonly docId: string; @@ -181,7 +179,7 @@ class MemoryBackend implements SyncBackend { hasOp(replicaHex: string, counter: number): boolean { return Array.from(this.opsByRefHex.values()).some( - (v) => bytesToHex(v.op.meta.id.replica) === replicaHex && v.op.meta.id.counter === counter, + (v) => bytesToHex(v.op.meta.id.replica) === replicaHex && v.op.meta.id.counter === counter ); } @@ -190,7 +188,7 @@ class MemoryBackend implements SyncBackend { } async listOpRefs(filter: Filter): Promise { - if ('all' in filter) { + if ("all" in filter) { return Array.from(this.opsByRefHex.values(), (v) => v.opRef); } @@ -222,7 +220,7 @@ class MemoryBackend implements SyncBackend { for (const { opRef, op } of entries) { switch (op.kind.type) { - case 'insert': { + case "insert": { const nodeHex = op.kind.node; const oldParentHex = parentByNodeHex.get(nodeHex); const newParentHex = op.kind.parent; @@ -238,7 +236,7 @@ class MemoryBackend implements SyncBackend { parentByNodeHex.set(nodeHex, newParentHex); break; } - case 'move': { + case "move": { const nodeHex = op.kind.node; const oldParentHex = parentByNodeHex.get(nodeHex); const newParentHex = op.kind.newParent; @@ -254,8 +252,8 @@ class MemoryBackend implements SyncBackend { parentByNodeHex.set(nodeHex, newParentHex); break; } - case 'delete': - case 'tombstone': { + case "delete": + case "tombstone": { const nodeHex = op.kind.node; const oldParentHex = parentByNodeHex.get(nodeHex); const newParentHex = TRASH_HEX; @@ -267,7 +265,7 @@ class MemoryBackend implements SyncBackend { parentByNodeHex.set(nodeHex, newParentHex); break; } - case 'payload': { + case "payload": { const nodeHex = op.kind.node; const parentHex = parentByNodeHex.get(nodeHex); if (parentHex === targetParentHex) add(opRef); @@ -287,7 +285,7 @@ class MemoryBackend implements SyncBackend { async getOpsByOpRefs(opRefs: OpRef[]): Promise { return opRefs.map((r) => { const stored = this.opsByRefHex.get(bytesToHex(r)); - if (!stored) throw new Error('opRef missing locally'); + if (!stored) throw new Error("opRef missing locally"); return stored.op; }); } @@ -307,20 +305,20 @@ class MemoryBackend implements SyncBackend { class FailingApplyBackend extends MemoryBackend { override async applyOps(_ops: Operation[]): Promise { - throw new Error('apply failed'); + throw new Error("apply failed"); } } -test('syncOnce does not starve macrotask transports', async () => { - const docId = 'doc-sync-macrotask'; - const root = '0'.repeat(32); +test("syncOnce does not starve macrotask transports", async () => { + const docId = "doc-sync-macrotask"; + const root = "0".repeat(32); const a = new MemoryBackend(docId); const b = new MemoryBackend(docId); await a.applyOps([ makeOp(replicas.a, 1, 1, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(1), orderKey: orderKeyFromPosition(0), @@ -337,21 +335,19 @@ test('syncOnce does not starve macrotask transports', async () => { await pa.syncOnce(ta, { all: {} }, { maxCodewords: 10_000, codewordsPerMessage: 256 }); - await waitUntil(() => b.hasOp(replicaHex.a, 1), { - message: 'expected b to receive a:1 via macrotask duplex', - }); + await waitUntil(() => b.hasOp(replicaHex.a, 1), { message: "expected b to receive a:1 via macrotask duplex" }); }); -test('syncOnce paces outbound codewords until delayed ribltStatus arrives', async () => { - const docId = 'doc-sync-delayed-status'; - const root = '0'.repeat(32); +test("syncOnce paces outbound codewords until delayed ribltStatus arrives", async () => { + const docId = "doc-sync-delayed-status"; + const root = "0".repeat(32); const a = new MemoryBackend(docId); const b = new MemoryBackend(docId); await a.applyOps([ makeOp(replicas.a, 1, 1, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(1), orderKey: orderKeyFromPosition(0), @@ -368,22 +364,20 @@ test('syncOnce paces outbound codewords until delayed ribltStatus arrives', asyn await pa.syncOnce(ta, { all: {} }, { maxCodewords: 1_024, codewordsPerMessage: 256 }); - await waitUntil(() => b.hasOp(replicaHex.a, 1), { - message: 'expected b to receive a:1 after delayed ribltStatus', - }); + await waitUntil(() => b.hasOp(replicaHex.a, 1), { message: "expected b to receive a:1 after delayed ribltStatus" }); }); -test('syncOnce protobuf roundtrips ribltStatus.more', () => { +test("syncOnce protobuf roundtrips ribltStatus.more", () => { const msg = { v: 0, - docId: 'doc-sync-more-codec', + docId: "doc-sync-more-codec", payload: { - case: 'ribltStatus' as const, + case: "ribltStatus" as const, value: { - filterId: 'f_more', + filterId: "f_more", round: 7, payload: { - case: 'more' as const, + case: "more" as const, value: { codewordsReceived: 9n, credits: 2 }, }, }, @@ -393,9 +387,9 @@ test('syncOnce protobuf roundtrips ribltStatus.more', () => { expect(treecrdtSyncV0ProtobufCodec.decode(treecrdtSyncV0ProtobufCodec.encode(msg))).toEqual(msg); }); -test('syncOnce waits for ribltStatus.more before sending another codeword batch', async () => { - const docId = 'doc-sync-more-flow-control'; - const root = '0'.repeat(32); +test("syncOnce waits for ribltStatus.more before sending another codeword batch", async () => { + const docId = "doc-sync-more-flow-control"; + const root = "0".repeat(32); const a = new MemoryBackend(docId); const b = new MemoryBackend(docId); @@ -404,11 +398,11 @@ test('syncOnce waits for ribltStatus.more before sending another codeword batch' for (let i = 1; i <= 12; i += 1) { ops.push( makeOp(replicas.a, i, i, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(i), orderKey: orderKeyFromPosition(i - 1), - }), + }) ); } await a.applyOps(ops); @@ -421,34 +415,74 @@ test('syncOnce waits for ribltStatus.more before sending another codeword batch' await pa.syncOnce(ta, { all: {} }, { maxCodewords: 4_096, codewordsPerMessage: 1 }); - await waitUntil(() => b.hasOp(replicaHex.a, 12), { - message: 'expected b to receive all ops after ribltStatus.more flow control', - }); + await waitUntil(() => b.hasOp(replicaHex.a, 12), { message: "expected b to receive all ops after ribltStatus.more flow control" }); const wire = log.map((entry) => `${entry.dir}:${entry.msg.payload.case}`); let sawMore = false; for (let i = 1; i < wire.length; i += 1) { - if (wire[i - 1] === 'bToA:ribltStatus') { - const status = - log[i - 1]!.msg.payload.case === 'ribltStatus' ? log[i - 1]!.msg.payload.value : null; - if (status?.payload.case === 'more') sawMore = true; + if (wire[i - 1] === "bToA:ribltStatus") { + const status = log[i - 1]!.msg.payload.case === "ribltStatus" ? log[i - 1]!.msg.payload.value : null; + if (status?.payload.case === "more") sawMore = true; } - expect(!(wire[i - 1] === 'aToB:ribltCodewords' && wire[i] === 'aToB:ribltCodewords')).toBe( - true, - ); + expect(!(wire[i - 1] === "aToB:ribltCodewords" && wire[i] === "aToB:ribltCodewords")).toBe(true); } expect(sawMore).toBe(true); }); -test('syncOnce rejects when local message handler throws during apply', async () => { - const docId = 'doc-sync-apply-error'; - const root = '0'.repeat(32); +test("syncOnce can direct-send a small clean-slate scope without riblt codewords", async () => { + const docId = "doc-sync-direct-send-small-scope"; + const root = "0".repeat(32); + + const a = new MemoryBackend(docId); + const b = new MemoryBackend(docId); + + await b.applyOps([ + makeOp(replicas.b, 1, 1, { + type: "insert", + parent: root, + node: nodeIdFromInt(1), + orderKey: orderKeyFromPosition(0), + }), + makeOp(replicas.b, 2, 2, { + type: "insert", + parent: root, + node: nodeIdFromInt(2), + orderKey: orderKeyFromPosition(1), + }), + ]); + + const [ta, tb, log] = createLoggedTimedDuplex>(); + const pa = new SyncPeer(a, { directSendThreshold: 8 }); + const pb = new SyncPeer(b, { directSendThreshold: 8 }); + pa.attach(ta); + pb.attach(tb); + + await pa.syncOnce(ta, { children: { parent: nodeIdToBytes16(root) } }, { + maxCodewords: 4_096, + codewordsPerMessage: 16, + }); + + await waitUntil(() => a.hasOp(replicaHex.b, 2), { + message: "expected clean-slate initiator to receive direct-sent scope", + }); + + const wire = log.map((entry) => `${entry.dir}:${entry.msg.payload.case}`); + expect(wire).toContain("aToB:hello"); + expect(wire).toContain("bToA:helloAck"); + expect(wire).toContain("bToA:opsBatch"); + expect(wire).not.toContain("aToB:ribltCodewords"); + expect(wire).not.toContain("bToA:ribltStatus"); +}); + +test("syncOnce rejects when local message handler throws during apply", async () => { + const docId = "doc-sync-apply-error"; + const root = "0".repeat(32); const a = new FailingApplyBackend(docId); const b = new MemoryBackend(docId); await b.applyOps([ makeOp(replicas.b, 1, 1, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(1), orderKey: orderKeyFromPosition(0), @@ -459,9 +493,7 @@ test('syncOnce rejects when local message handler throws during apply', async () try { const timed = Promise.race([ pa.syncOnce(ta, { all: {} }, { maxCodewords: 10_000, codewordsPerMessage: 256 }), - new Promise((_, reject) => - setTimeout(() => reject(new Error('syncOnce timed out')), 2_000), - ), + new Promise((_, reject) => setTimeout(() => reject(new Error("syncOnce timed out")), 2_000)), ]); await expect(timed).rejects.toThrow(/apply failed/i); } finally { @@ -469,40 +501,20 @@ test('syncOnce rejects when local message handler throws during apply', async () } }); -test('sync all converges union of opRefs', async () => { - const docId = 'doc-sync-all'; - const root = '0'.repeat(32); +test("sync all converges union of opRefs", async () => { + const docId = "doc-sync-all"; + const root = "0".repeat(32); const a = new MemoryBackend(docId); const b = new MemoryBackend(docId); await a.applyOps([ - makeOp(replicas.a, 1, 1, { - type: 'insert', - parent: root, - node: nodeIdFromInt(1), - orderKey: orderKeyFromPosition(0), - }), - makeOp(replicas.a, 2, 2, { - type: 'insert', - parent: root, - node: nodeIdFromInt(2), - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.a, 1, 1, { type: "insert", parent: root, node: nodeIdFromInt(1), orderKey: orderKeyFromPosition(0) }), + makeOp(replicas.a, 2, 2, { type: "insert", parent: root, node: nodeIdFromInt(2), orderKey: orderKeyFromPosition(0) }), ]); await b.applyOps([ - makeOp(replicas.b, 1, 3, { - type: 'insert', - parent: root, - node: nodeIdFromInt(3), - orderKey: orderKeyFromPosition(0), - }), - makeOp(replicas.a, 2, 2, { - type: 'insert', - parent: root, - node: nodeIdFromInt(2), - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.b, 1, 3, { type: "insert", parent: root, node: nodeIdFromInt(3), orderKey: orderKeyFromPosition(0) }), + makeOp(replicas.a, 2, 2, { type: "insert", parent: root, node: nodeIdFromInt(2), orderKey: orderKeyFromPosition(0) }), ]); const { peerA: pa, transportA: ta, detach } = createPeers(a, b); @@ -518,9 +530,9 @@ test('sync all converges union of opRefs', async () => { } }); -test('sync all transfers a single missing op (hole in the middle)', async () => { - const docId = 'doc-sync-one-missing'; - const root = '0'.repeat(32); +test("sync all transfers a single missing op (hole in the middle)", async () => { + const docId = "doc-sync-one-missing"; + const root = "0".repeat(32); const a = new MemoryBackend(docId); const b = new MemoryBackend(docId); @@ -531,11 +543,11 @@ test('sync all transfers a single missing op (hole in the middle)', async () => for (let counter = 1; counter <= size; counter += 1) { ops.push( makeOp(replicas.s, counter, counter, { - type: 'insert', + type: "insert", parent: root, node: nodeIdFromInt(counter), orderKey: orderKeyFromPosition(counter - 1), - }), + }) ); } @@ -556,42 +568,22 @@ test('sync all transfers a single missing op (hole in the middle)', async () => } }); -test('sync children(parent) only transfers those children', async () => { - const docId = 'doc-sync-children'; - const parentAHex = 'a0'.repeat(16); - const parentBHex = 'b0'.repeat(16); +test("sync children(parent) only transfers those children", async () => { + const docId = "doc-sync-children"; + const parentAHex = "a0".repeat(16); + const parentBHex = "b0".repeat(16); const parentABytes = nodeIdToBytes16(parentAHex); const a = new MemoryBackend(docId); const b = new MemoryBackend(docId); await a.applyOps([ - makeOp(replicas.a, 1, 1, { - type: 'insert', - parent: parentAHex, - node: nodeIdFromInt(1), - orderKey: orderKeyFromPosition(0), - }), - makeOp(replicas.a, 2, 2, { - type: 'insert', - parent: parentBHex, - node: nodeIdFromInt(2), - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.a, 1, 1, { type: "insert", parent: parentAHex, node: nodeIdFromInt(1), orderKey: orderKeyFromPosition(0) }), + makeOp(replicas.a, 2, 2, { type: "insert", parent: parentBHex, node: nodeIdFromInt(2), orderKey: orderKeyFromPosition(0) }), ]); await b.applyOps([ - makeOp(replicas.b, 1, 3, { - type: 'insert', - parent: parentAHex, - node: nodeIdFromInt(3), - orderKey: orderKeyFromPosition(0), - }), - makeOp(replicas.b, 2, 4, { - type: 'insert', - parent: parentBHex, - node: nodeIdFromInt(4), - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.b, 1, 3, { type: "insert", parent: parentAHex, node: nodeIdFromInt(3), orderKey: orderKeyFromPosition(0) }), + makeOp(replicas.b, 2, 4, { type: "insert", parent: parentBHex, node: nodeIdFromInt(4), orderKey: orderKeyFromPosition(0) }), ]); const { peerA: pa, transportA: ta, detach } = createPeers(a, b); @@ -599,7 +591,7 @@ test('sync children(parent) only transfers those children', async () => { await pa.syncOnce( ta, { children: { parent: parentABytes } }, - { maxCodewords: 10_000, codewordsPerMessage: 256 }, + { maxCodewords: 10_000, codewordsPerMessage: 256 } ); await tick(); @@ -618,10 +610,10 @@ test('sync children(parent) only transfers those children', async () => { } }); -test('sync children(parent) includes boundary-crossing moves', async () => { - const docId = 'doc-sync-boundary-move'; - const parentAHex = 'a0'.repeat(16); - const parentBHex = 'b0'.repeat(16); +test("sync children(parent) includes boundary-crossing moves", async () => { + const docId = "doc-sync-boundary-move"; + const parentAHex = "a0".repeat(16); + const parentBHex = "b0".repeat(16); const parentABytes = nodeIdToBytes16(parentAHex); const a = new MemoryBackend(docId); @@ -630,20 +622,10 @@ test('sync children(parent) includes boundary-crossing moves', async () => { const node = nodeIdFromInt(0x10); await a.applyOps([ - makeOp(replicas.a, 1, 1, { - type: 'insert', - parent: parentAHex, - node, - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.a, 1, 1, { type: "insert", parent: parentAHex, node, orderKey: orderKeyFromPosition(0) }), // Move the node out of the subtree. The move is still relevant to `children(parentA)` // because it changes the canonical child set of `parentA`. - makeOp(replicas.a, 2, 2, { - type: 'move', - node, - newParent: parentBHex, - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.a, 2, 2, { type: "move", node, newParent: parentBHex, orderKey: orderKeyFromPosition(0) }), ]); const { peerA: pa, transportA: ta, detach } = createPeers(a, b); @@ -651,7 +633,7 @@ test('sync children(parent) includes boundary-crossing moves', async () => { await pa.syncOnce( ta, { children: { parent: parentABytes } }, - { maxCodewords: 10_000, codewordsPerMessage: 256 }, + { maxCodewords: 10_000, codewordsPerMessage: 256 } ); await tick(); @@ -665,10 +647,10 @@ test('sync children(parent) includes boundary-crossing moves', async () => { } }); -test('sync children(parent) includes latest payload when node moves into parent', async () => { - const docId = 'doc-sync-children-payload'; - const parentAHex = 'a0'.repeat(16); - const parentBHex = 'b0'.repeat(16); +test("sync children(parent) includes latest payload when node moves into parent", async () => { + const docId = "doc-sync-children-payload"; + const parentAHex = "a0".repeat(16); + const parentBHex = "b0".repeat(16); const parentBBytes = nodeIdToBytes16(parentBHex); const a = new MemoryBackend(docId); @@ -678,19 +660,9 @@ test('sync children(parent) includes latest payload when node moves into parent' const payload = new Uint8Array([1, 2, 3]); await a.applyOps([ - makeOp(replicas.a, 1, 1, { - type: 'insert', - parent: parentAHex, - node, - orderKey: orderKeyFromPosition(0), - }), - makeOp(replicas.a, 2, 2, { type: 'payload', node, payload }), - makeOp(replicas.a, 3, 3, { - type: 'move', - node, - newParent: parentBHex, - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.a, 1, 1, { type: "insert", parent: parentAHex, node, orderKey: orderKeyFromPosition(0) }), + makeOp(replicas.a, 2, 2, { type: "payload", node, payload }), + makeOp(replicas.a, 3, 3, { type: "move", node, newParent: parentBHex, orderKey: orderKeyFromPosition(0) }), ]); const { peerA: pa, transportA: ta, detach } = createPeers(a, b); @@ -698,7 +670,7 @@ test('sync children(parent) includes latest payload when node moves into parent' await pa.syncOnce( ta, { children: { parent: parentBBytes } }, - { maxCodewords: 10_000, codewordsPerMessage: 256 }, + { maxCodewords: 10_000, codewordsPerMessage: 256 } ); await tick(); @@ -711,51 +683,37 @@ test('sync children(parent) includes latest payload when node moves into parent' expect(setHex(aChildrenB)).toEqual(setHex(bChildrenB)); const ops = await b.getOpsByOpRefs(bChildrenB); - const payloadOps = ops.filter((op) => op.kind.type === 'payload' && op.kind.payload !== null); + const payloadOps = ops.filter((op) => op.kind.type === "payload" && op.kind.payload !== null); expect(payloadOps.length).toBe(1); - expect(payloadOps[0]?.kind.type).toBe('payload'); + expect(payloadOps[0]?.kind.type).toBe("payload"); expect(payloadOps[0]?.kind.payload).toEqual(payload); } finally { detach(); } }); -test('subscribe keeps peers converging (push deltas)', async () => { - const docId = 'doc-subscribe'; - const root = '0'.repeat(32); +test("subscribe keeps peers converging (push deltas)", async () => { + const docId = "doc-subscribe"; + const root = "0".repeat(32); const a = new MemoryBackend(docId); const b = new MemoryBackend(docId); await a.applyOps([ - makeOp(replicas.a, 1, 1, { - type: 'insert', - parent: root, - node: nodeIdFromInt(1), - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.a, 1, 1, { type: "insert", parent: root, node: nodeIdFromInt(1), orderKey: orderKeyFromPosition(0) }), ]); const { peerA: pa, peerB: pb, transportA: ta, detach } = createPeers(a, b); try { const sub = pa.subscribe(ta, { all: {} }, { maxCodewords: 10_000, codewordsPerMessage: 256 }); try { - await waitUntil(() => b.hasOp(replicaHex.a, 1), { - message: 'expected b to receive a:1 via subscription', - }); + await waitUntil(() => b.hasOp(replicaHex.a, 1), { message: "expected b to receive a:1 via subscription" }); await b.applyOps([ - makeOp(replicas.b, 1, 2, { - type: 'insert', - parent: root, - node: nodeIdFromInt(2), - orderKey: orderKeyFromPosition(0), - }), + makeOp(replicas.b, 1, 2, { type: "insert", parent: root, node: nodeIdFromInt(2), orderKey: orderKeyFromPosition(0) }), ]); await pb.notifyLocalUpdate(); - await waitUntil(() => a.hasOp(replicaHex.b, 1), { - message: 'expected a to receive b:1 via subscription', - }); + await waitUntil(() => a.hasOp(replicaHex.b, 1), { message: "expected a to receive b:1 via subscription" }); } finally { sub.stop(); await sub.done; diff --git a/packages/sync/server/postgres-node/src/cli.ts b/packages/sync/server/postgres-node/src/cli.ts index 06980501..796751a3 100644 --- a/packages/sync/server/postgres-node/src/cli.ts +++ b/packages/sync/server/postgres-node/src/cli.ts @@ -1,19 +1,19 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; -import { base64urlDecode } from '@treecrdt/auth'; +import { base64urlDecode } from "@treecrdt/auth"; -import { startSyncServer } from './server.js'; +import { startSyncServer } from "./server.js"; -const LOCAL_POSTGRES_URL_EXAMPLE = 'postgres://postgres:postgres@127.0.0.1:5432/postgres'; +const LOCAL_POSTGRES_URL_EXAMPLE = "postgres://postgres:postgres@127.0.0.1:5432/postgres"; function parseBooleanEnv(name: string, fallback: boolean): boolean { const raw = process.env[name]; if (!raw || raw.trim().length === 0) return fallback; const normalized = raw.trim().toLowerCase(); - if (['1', 'true', 'yes', 'on'].includes(normalized)) return true; - if (['0', 'false', 'no', 'off'].includes(normalized)) return false; + if (["1", "true", "yes", "on"].includes(normalized)) return true; + if (["0", "false", "no", "off"].includes(normalized)) return false; throw new Error(`${name} must be a boolean (true/false), got: ${raw}`); } @@ -21,16 +21,14 @@ function parseIssuerPublicKeysEnv(name: string): Uint8Array[] { const raw = process.env[name]; if (!raw || raw.trim().length === 0) return []; return raw - .split(',') + .split(",") .map((entry) => entry.trim()) .filter((entry) => entry.length > 0) .map((entry) => { try { return base64urlDecode(entry); } catch (err) { - throw new Error( - `${name} has invalid base64url key "${entry}": ${err instanceof Error ? err.message : String(err)}`, - ); + throw new Error(`${name} has invalid base64url key "${entry}": ${err instanceof Error ? err.message : String(err)}`); } }); } @@ -39,17 +37,17 @@ function buildPostgresUrlFromParts(): string | undefined { const host = process.env.TREECRDT_POSTGRES_HOST?.trim(); const db = process.env.TREECRDT_POSTGRES_DB?.trim(); const user = process.env.TREECRDT_POSTGRES_USER?.trim(); - const password = process.env.TREECRDT_POSTGRES_PASSWORD ?? ''; + const password = process.env.TREECRDT_POSTGRES_PASSWORD ?? ""; const portRaw = process.env.TREECRDT_POSTGRES_PORT?.trim(); if (!host && !db && !user && !password && !portRaw) return undefined; if (!host || !db || !user) { throw new Error( - 'TREECRDT_POSTGRES_HOST, TREECRDT_POSTGRES_DB, and TREECRDT_POSTGRES_USER are required when TREECRDT_POSTGRES_URL is not set', + "TREECRDT_POSTGRES_HOST, TREECRDT_POSTGRES_DB, and TREECRDT_POSTGRES_USER are required when TREECRDT_POSTGRES_URL is not set" ); } - const port = Number(portRaw ?? '5432'); - if (!Number.isFinite(port) || port <= 0) throw new Error('invalid TREECRDT_POSTGRES_PORT'); + const port = Number(portRaw ?? "5432"); + if (!Number.isFinite(port) || port <= 0) throw new Error("invalid TREECRDT_POSTGRES_PORT"); const encodedUser = encodeURIComponent(user); const encodedPassword = encodeURIComponent(password); @@ -58,82 +56,79 @@ function buildPostgresUrlFromParts(): string | undefined { function clientHostForBindHost(host: string): string { const trimmed = host.trim(); - if (trimmed === '0.0.0.0' || trimmed === '::' || trimmed === '[::]') { - return 'localhost'; + if (trimmed === "0.0.0.0" || trimmed === "::" || trimmed === "[::]") { + return "localhost"; } return trimmed; } function readPackageVersion(): string | undefined { try { - const packageJsonUrl = new URL('../package.json', import.meta.url); - const parsed = JSON.parse(fs.readFileSync(packageJsonUrl, 'utf8')) as { version?: unknown }; - return typeof parsed.version === 'string' && parsed.version.trim().length > 0 - ? parsed.version.trim() - : undefined; + const packageJsonUrl = new URL("../package.json", import.meta.url); + const parsed = JSON.parse(fs.readFileSync(packageJsonUrl, "utf8")) as { version?: unknown }; + return typeof parsed.version === "string" && parsed.version.trim().length > 0 ? parsed.version.trim() : undefined; } catch { return undefined; } } async function main() { - const host = process.env.HOST ?? '0.0.0.0'; - const port = Number(process.env.PORT ?? '8787'); + const host = process.env.HOST ?? "0.0.0.0"; + const port = Number(process.env.PORT ?? "8787"); const postgresUrl = process.env.TREECRDT_POSTGRES_URL?.trim() || buildPostgresUrlFromParts(); - const idleCloseMs = Number(process.env.TREECRDT_IDLE_CLOSE_MS ?? '30000'); - const maxPayloadBytes = Number( - process.env.TREECRDT_MAX_PAYLOAD_BYTES ?? String(10 * 1024 * 1024), - ); + const maxCodewords = Number(process.env.TREECRDT_SYNC_MAX_CODEWORDS ?? "0"); + const directSendThreshold = Number(process.env.TREECRDT_SYNC_DIRECT_SEND_THRESHOLD ?? "0"); + const idleCloseMs = Number(process.env.TREECRDT_IDLE_CLOSE_MS ?? "30000"); + const maxPayloadBytes = Number(process.env.TREECRDT_MAX_PAYLOAD_BYTES ?? String(10 * 1024 * 1024)); const authToken = process.env.TREECRDT_SYNC_AUTH_TOKEN?.trim() || undefined; - const authCapabilityIssuerPublicKeys = parseIssuerPublicKeysEnv( - 'TREECRDT_SYNC_CWT_ISSUER_PUBKEYS', - ); + const authCapabilityIssuerPublicKeys = parseIssuerPublicKeysEnv("TREECRDT_SYNC_CWT_ISSUER_PUBKEYS"); const docIdPattern = process.env.TREECRDT_DOC_ID_PATTERN?.trim() || undefined; - const allowDocCreate = parseBooleanEnv('TREECRDT_ALLOW_DOC_CREATE', true); - const enablePgNotify = parseBooleanEnv('TREECRDT_PG_NOTIFY_ENABLED', true); - const pgNotifyChannel = - process.env.TREECRDT_PG_NOTIFY_CHANNEL?.trim() || 'treecrdt_sync_doc_updates'; - const rateLimitMaxUpgrades = Number(process.env.TREECRDT_RATE_LIMIT_MAX_UPGRADES ?? '0'); - const rateLimitWindowMs = Number(process.env.TREECRDT_RATE_LIMIT_WINDOW_MS ?? '60000'); + const allowDocCreate = parseBooleanEnv("TREECRDT_ALLOW_DOC_CREATE", true); + const enablePgNotify = parseBooleanEnv("TREECRDT_PG_NOTIFY_ENABLED", true); + const pgNotifyChannel = process.env.TREECRDT_PG_NOTIFY_CHANNEL?.trim() || "treecrdt_sync_doc_updates"; + const rateLimitMaxUpgrades = Number(process.env.TREECRDT_RATE_LIMIT_MAX_UPGRADES ?? "0"); + const rateLimitWindowMs = Number(process.env.TREECRDT_RATE_LIMIT_WINDOW_MS ?? "60000"); const packageVersion = readPackageVersion(); const gitSha = process.env.TREECRDT_SYNC_GIT_SHA?.trim() || undefined; - const gitDirty = parseBooleanEnv('TREECRDT_SYNC_GIT_DIRTY', false); + const gitDirty = parseBooleanEnv("TREECRDT_SYNC_GIT_DIRTY", false); const startedAt = new Date().toISOString(); const backendModule = process.env.TREECRDT_POSTGRES_BACKEND_MODULE ?? - path.resolve( - path.dirname(fileURLToPath(import.meta.url)), - '../../../treecrdt-postgres-napi/dist/index.js', - ); + path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../treecrdt-postgres-napi/dist/index.js"); if (!postgresUrl || postgresUrl.length === 0) { throw new Error( [ - 'missing Postgres connection for sync-server:postgres', - 'set TREECRDT_POSTGRES_URL or TREECRDT_POSTGRES_HOST/PORT/DB/USER/PASSWORD', + "missing Postgres connection for sync-server:postgres", + "set TREECRDT_POSTGRES_URL or TREECRDT_POSTGRES_HOST/PORT/DB/USER/PASSWORD", `local example: TREECRDT_POSTGRES_URL=${LOCAL_POSTGRES_URL_EXAMPLE} pnpm sync-server:postgres`, - 'or use: pnpm sync-server:postgres:local', - ].join('\n'), + "or use: pnpm sync-server:postgres:local", + ].join("\n") ); } if (!Number.isFinite(port) || port <= 0) throw new Error(`invalid PORT: ${process.env.PORT}`); - if (!Number.isFinite(idleCloseMs) || idleCloseMs < 0) - throw new Error('invalid TREECRDT_IDLE_CLOSE_MS'); + if (!Number.isFinite(maxCodewords) || maxCodewords < 0) throw new Error("invalid TREECRDT_SYNC_MAX_CODEWORDS"); + if (!Number.isFinite(directSendThreshold) || directSendThreshold < 0) { + throw new Error("invalid TREECRDT_SYNC_DIRECT_SEND_THRESHOLD"); + } + if (!Number.isFinite(idleCloseMs) || idleCloseMs < 0) throw new Error("invalid TREECRDT_IDLE_CLOSE_MS"); if (!Number.isFinite(maxPayloadBytes) || maxPayloadBytes <= 0) { - throw new Error('invalid TREECRDT_MAX_PAYLOAD_BYTES'); + throw new Error("invalid TREECRDT_MAX_PAYLOAD_BYTES"); } if (!Number.isFinite(rateLimitMaxUpgrades) || rateLimitMaxUpgrades < 0) { - throw new Error('invalid TREECRDT_RATE_LIMIT_MAX_UPGRADES'); + throw new Error("invalid TREECRDT_RATE_LIMIT_MAX_UPGRADES"); } if (!Number.isFinite(rateLimitWindowMs) || rateLimitWindowMs <= 0) { - throw new Error('invalid TREECRDT_RATE_LIMIT_WINDOW_MS'); + throw new Error("invalid TREECRDT_RATE_LIMIT_WINDOW_MS"); } const handle = await startSyncServer({ host, port, postgresUrl, + maxCodewords: maxCodewords > 0 ? maxCodewords : undefined, + directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, idleCloseMs, maxPayloadBytes, backendModule, @@ -159,16 +154,14 @@ async function main() { console.log(`- sync endpoint: ws://${clientHost}:${handle.port}/sync?docId=YOUR_DOC_ID`); console.log(`- backend module: ${handle.backendModule}`); if (packageVersion) console.log(`- version: ${packageVersion}`); - if (gitSha) console.log(`- git sha: ${gitSha}${gitDirty ? ' (dirty)' : ''}`); + if (gitSha) console.log(`- git sha: ${gitSha}${gitDirty ? " (dirty)" : ""}`); if (authCapabilityIssuerPublicKeys.length > 0) { - console.log( - `- auth: capability CWT enabled (${authCapabilityIssuerPublicKeys.length} issuer keys)`, - ); + console.log(`- auth: capability CWT enabled (${authCapabilityIssuerPublicKeys.length} issuer keys)`); } else if (authToken) { - console.log('- auth: static token enabled (bearer token or ?token=...)'); + console.log("- auth: static token enabled (bearer token or ?token=...)"); } if (docIdPattern) console.log(`- docId policy: ${docIdPattern}`); - if (!allowDocCreate) console.log('- doc creation policy: deny unknown docId'); + if (!allowDocCreate) console.log("- doc creation policy: deny unknown docId"); if (enablePgNotify) console.log(`- pg notify: enabled on channel ${pgNotifyChannel}`); if (rateLimitMaxUpgrades > 0) { console.log(`- rate limit: ${rateLimitMaxUpgrades} upgrades per ${rateLimitWindowMs}ms per IP`); diff --git a/packages/sync/server/postgres-node/src/server.ts b/packages/sync/server/postgres-node/src/server.ts index 0eca503d..a327c7a7 100644 --- a/packages/sync/server/postgres-node/src/server.ts +++ b/packages/sync/server/postgres-node/src/server.ts @@ -1,23 +1,29 @@ -import path from 'node:path'; -import { randomUUID } from 'node:crypto'; -import { pathToFileURL } from 'node:url'; - -import { base64urlDecode, describeTreecrdtCapabilityTokenV1 } from '@treecrdt/auth'; -import type { Operation } from '@treecrdt/interface'; -import { createReplayOnlySyncAuth } from '@treecrdt/sync'; -import type { SyncBackend, SyncPeer, SyncPeerOptions } from '@treecrdt/sync'; -import { createCapabilityMaterialStore, createOpAuthStore } from '@treecrdt/sync-postgres'; -import type { PostgresCapabilityMaterialStore, PostgresOpAuthStore } from '@treecrdt/sync-postgres'; -import { treecrdtSyncV0ProtobufCodec } from '@treecrdt/sync/protobuf'; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { pathToFileURL } from "node:url"; + +import { base64urlDecode, describeTreecrdtCapabilityTokenV1 } from "@treecrdt/auth"; +import type { Operation } from "@treecrdt/interface"; +import { createReplayOnlySyncAuth } from "@treecrdt/sync"; +import type { SyncBackend, SyncPeer, SyncPeerOptions } from "@treecrdt/sync"; +import { + createCapabilityMaterialStore, + createOpAuthStore, +} from "@treecrdt/sync-postgres"; +import type { + PostgresCapabilityMaterialStore, + PostgresOpAuthStore, +} from "@treecrdt/sync-postgres"; +import { treecrdtSyncV0ProtobufCodec } from "@treecrdt/sync/protobuf"; import type { WebSocketSyncServerDocHandle, WebSocketSyncServerDocProvider, WebSocketSyncServerHooks, WebSocketSyncServerUpgradeContext, WebSocketSyncServerUpgradeHook, -} from '@treecrdt/sync-server-core'; -import { startWebSocketSyncServer } from '@treecrdt/sync-server-core'; -import { Client as PgClient } from 'pg'; +} from "@treecrdt/sync-server-core"; +import { startWebSocketSyncServer } from "@treecrdt/sync-server-core"; +import { Client as PgClient } from "pg"; type Awaitable = T | Promise; @@ -42,6 +48,8 @@ export type SyncServerOptions = { port?: number; postgresUrl: string; backendModule?: string; + maxCodewords?: number; + directSendThreshold?: number; idleCloseMs?: number; maxPayloadBytes?: number; authToken?: string; @@ -96,8 +104,14 @@ type PostgresDocUpdateBusOptions = { onDocUpdate: (docId: string) => void; }; -export { createCapabilityMaterialStore, createOpAuthStore }; -export type { PostgresCapabilityMaterialStore, PostgresOpAuthStore }; +export { + createCapabilityMaterialStore, + createOpAuthStore, +}; +export type { + PostgresCapabilityMaterialStore, + PostgresOpAuthStore, +}; function ensurePostgresChannelName(value: string): string { const trimmed = value.trim(); @@ -109,11 +123,11 @@ function ensurePostgresChannelName(value: string): string { function describeAuthMode( authToken: string | undefined, - issuerPublicKeys: Uint8Array[], -): 'none' | 'static_token' | 'capability_cwt' { - if (issuerPublicKeys.length > 0) return 'capability_cwt'; - if (authToken) return 'static_token'; - return 'none'; + issuerPublicKeys: Uint8Array[] +): "none" | "static_token" | "capability_cwt" { + if (issuerPublicKeys.length > 0) return "capability_cwt"; + if (authToken) return "static_token"; + return "none"; } function errorMessage(error: unknown): string { @@ -132,14 +146,14 @@ async function withTimeout(promise: Promise, ms: number, label: string): P (error) => { clearTimeout(timer); reject(error); - }, + } ); }); } function parseDocIdRegex(input: string | RegExp | undefined): RegExp | undefined { if (!input) return undefined; - if (input instanceof RegExp) return new RegExp(input.source, input.flags.replace(/[gy]/g, '')); + if (input instanceof RegExp) return new RegExp(input.source, input.flags.replace(/[gy]/g, "")); const trimmed = input.trim(); if (trimmed.length === 0) return undefined; return new RegExp(trimmed); @@ -173,7 +187,7 @@ class PostgresDocUpdateBus { private async start(): Promise { await Promise.all([this.queryClient.connect(), this.listenClient.connect()]); - this.listenClient.on('notification', (msg: { channel: string; payload?: string }) => { + this.listenClient.on("notification", (msg: { channel: string; payload?: string }) => { if (msg.channel !== this.channel) return; if (!msg.payload) return; let payload: DocUpdatePayload; @@ -182,7 +196,7 @@ class PostgresDocUpdateBus { } catch { payload = { docId: msg.payload }; } - if (!payload || typeof payload.docId !== 'string' || payload.docId.length === 0) return; + if (!payload || typeof payload.docId !== "string" || payload.docId.length === 0) return; if (payload.source && payload.source === this.sourceId) return; this.opts.onDocUpdate(payload.docId); }); @@ -191,21 +205,18 @@ class PostgresDocUpdateBus { } async hasDoc(docId: string): Promise { - const res = await this.queryClient.query( - 'SELECT 1 FROM treecrdt_meta WHERE doc_id = $1 LIMIT 1', - [docId], - ); + const res = await this.queryClient.query("SELECT 1 FROM treecrdt_meta WHERE doc_id = $1 LIMIT 1", [docId]); return (res.rowCount ?? 0) > 0; } async publishDocUpdate(docId: string): Promise { if (this.closed || docId.length === 0) return; const payload = JSON.stringify({ docId, source: this.sourceId } satisfies DocUpdatePayload); - await this.queryClient.query('SELECT pg_notify($1, $2)', [this.channel, payload]); + await this.queryClient.query("SELECT pg_notify($1, $2)", [this.channel, payload]); } async ping(): Promise { - await this.queryClient.query('SELECT 1'); + await this.queryClient.query("SELECT 1"); } async close(): Promise { @@ -221,10 +232,10 @@ class PostgresDocUpdateBus { } function isDeniedDecision( - decision: void | boolean | { allow: boolean; statusCode?: number; body?: string }, + decision: void | boolean | { allow: boolean; statusCode?: number; body?: string } ): decision is false | { allow: false; statusCode?: number; body?: string } { if (decision === false) return true; - if (typeof decision !== 'object' || decision === null) return false; + if (typeof decision !== "object" || decision === null) return false; return decision.allow === false; } @@ -243,11 +254,11 @@ function combineUpgradeHooks( } function extractAuthToken(ctx: WebSocketSyncServerUpgradeContext): string | null { - const queryToken = ctx.url.searchParams.get('token')?.trim(); + const queryToken = ctx.url.searchParams.get("token")?.trim(); if (queryToken) return queryToken; const rawAuth = ctx.req.headers.authorization; - if (typeof rawAuth === 'string') { + if (typeof rawAuth === "string") { const match = rawAuth.match(/^Bearer\s+(.+)$/i); return match?.[1]?.trim() || null; } @@ -261,21 +272,19 @@ function createStaticTokenAuthHook(expectedToken: string): WebSocketSyncServerUp return { allow: false, statusCode: 401, - body: 'missing or invalid auth token', + body: "missing or invalid auth token", }; }; } -function createCapabilityTokenAuthHook( - issuerPublicKeys: Uint8Array[], -): WebSocketSyncServerUpgradeHook { +function createCapabilityTokenAuthHook(issuerPublicKeys: Uint8Array[]): WebSocketSyncServerUpgradeHook { return async (ctx) => { const token = extractAuthToken(ctx); if (!token) { return { allow: false, statusCode: 401, - body: 'missing capability token', + body: "missing capability token", }; } try { @@ -302,29 +311,24 @@ function createDocIdPatternHook(pattern: RegExp): WebSocketSyncServerUpgradeHook return { allow: false, statusCode: 400, - body: 'invalid docId format', + body: "invalid docId format", }; }; } -function createKnownDocHook( - hasDoc: (docId: string) => Promise, -): WebSocketSyncServerUpgradeHook { +function createKnownDocHook(hasDoc: (docId: string) => Promise): WebSocketSyncServerUpgradeHook { return async (ctx) => { const known = await hasDoc(ctx.docId); if (known) return { allow: true }; return { allow: false, statusCode: 403, - body: 'docId creation disabled', + body: "docId creation disabled", }; }; } -function createIpRateLimitHook( - maxUpgrades: number, - windowMs: number, -): WebSocketSyncServerUpgradeHook { +function createIpRateLimitHook(maxUpgrades: number, windowMs: number): WebSocketSyncServerUpgradeHook { const buckets = new Map(); let lastPrunedAt = 0; return (ctx) => { @@ -337,7 +341,7 @@ function createIpRateLimitHook( } } - const key = ctx.remoteAddress ?? 'unknown'; + const key = ctx.remoteAddress ?? "unknown"; const bucket = buckets.get(key); if (!bucket || now - bucket.startedAt >= windowMs) { buckets.set(key, { startedAt: now, count: 1 }); @@ -350,30 +354,28 @@ function createIpRateLimitHook( return { allow: false, statusCode: 429, - body: 'too many upgrade requests', + body: "too many upgrade requests", }; }; } function ensureNonEmptyString(name: string, value: unknown): string { - if (typeof value !== 'string' || value.trim().length === 0) { + if (typeof value !== "string" || value.trim().length === 0) { throw new Error(`${name} must be a non-empty string`); } return value; } function moduleSpecifier(input: string): string { - if (input.startsWith('file://')) return input; - if (input.startsWith('.') || input.startsWith('/') || input.startsWith('\\')) { + if (input.startsWith("file://")) return input; + if (input.startsWith(".") || input.startsWith("/") || input.startsWith("\\")) { return pathToFileURL(path.resolve(input)).href; } return input; } -export async function loadPostgresBackendModule( - moduleName: string, -): Promise { - const name = ensureNonEmptyString('backendModule', moduleName); +export async function loadPostgresBackendModule(moduleName: string): Promise { + const name = ensureNonEmptyString("backendModule", moduleName); const specifier = moduleSpecifier(name); let imported: unknown; @@ -385,20 +387,17 @@ export async function loadPostgresBackendModule( } const mod = imported as Partial | null; - if (!mod || typeof mod.createPostgresNapiSyncBackendFactory !== 'function') { + if (!mod || typeof mod.createPostgresNapiSyncBackendFactory !== "function") { throw new Error( - `backend module "${name}" does not export createPostgresNapiSyncBackendFactory(url)`, + `backend module "${name}" does not export createPostgresNapiSyncBackendFactory(url)` ); } return { createPostgresNapiSyncBackendFactory: mod.createPostgresNapiSyncBackendFactory }; } -export function createPostgresNodeDocStore( - opts: PostgresNodeDocStoreOptions, -): PostgresNodeDocStore { +export function createPostgresNodeDocStore(opts: PostgresNodeDocStoreOptions): PostgresNodeDocStore { const idleCloseMs = Number(opts.idleCloseMs ?? 30_000); - if (!Number.isFinite(idleCloseMs) || idleCloseMs < 0) - throw new Error(`invalid idleCloseMs: ${opts.idleCloseMs}`); + if (!Number.isFinite(idleCloseMs) || idleCloseMs < 0) throw new Error(`invalid idleCloseMs: ${opts.idleCloseMs}`); const docs = new Map(); const openInFlight = new Map>(); @@ -445,10 +444,7 @@ export function createPostgresNodeDocStore( }, idleCloseMs); }; - const wrapBackendForContext = ( - ctx: DocContext, - backend: SyncBackend, - ): SyncBackend => { + const wrapBackendForContext = (ctx: DocContext, backend: SyncBackend): SyncBackend => { const applyOps = async (ops: Operation[]) => { if (ops.length === 0) return; const run = ctx.applyQueue.then(async () => { @@ -472,7 +468,7 @@ export function createPostgresNodeDocStore( }; const openContext = async (docId: string): Promise => { - if (closing) throw new Error('doc store is closing'); + if (closing) throw new Error("doc store is closing"); const existing = docs.get(docId); if (existing) return existing; @@ -491,7 +487,7 @@ export function createPostgresNodeDocStore( } if (closing) { await closeBackend(rawBackend); - throw new Error('doc store is closing'); + throw new Error("doc store is closing"); } const alreadyOpened = docs.get(docId); @@ -524,9 +520,9 @@ export function createPostgresNodeDocStore( return { provider: { open: async (docId: string): Promise> => { - const cleanDocId = ensureNonEmptyString('docId', docId); + const cleanDocId = ensureNonEmptyString("docId", docId); const ctx = await openContext(cleanDocId); - if (closing || ctx.closed) throw new Error('doc store is closing'); + if (closing || ctx.closed) throw new Error("doc store is closing"); ctx.connections += 1; if (ctx.closeTimer) { @@ -570,12 +566,12 @@ export function createPostgresNodeDocStore( async function createReadinessProbe( postgresUrl: string, - docUpdateBus: PostgresDocUpdateBus | undefined, + docUpdateBus: PostgresDocUpdateBus | undefined ): Promise { if (docUpdateBus) { return { check: async () => { - await withTimeout(docUpdateBus.ping(), 3_000, 'postgres readiness ping'); + await withTimeout(docUpdateBus.ping(), 3_000, "postgres readiness ping"); }, }; } @@ -584,7 +580,7 @@ async function createReadinessProbe( await client.connect(); return { check: async () => { - await withTimeout(client.query('SELECT 1'), 3_000, 'postgres readiness ping'); + await withTimeout(client.query("SELECT 1"), 3_000, "postgres readiness ping"); }, close: async () => { await client.end(); @@ -593,31 +589,31 @@ async function createReadinessProbe( } export async function startSyncServer(opts: SyncServerOptions): Promise { - const host = opts.host ?? '0.0.0.0'; + const host = opts.host ?? "0.0.0.0"; const port = Number(opts.port ?? 8787); const backendModule = ensureNonEmptyString( - 'backendModule', - opts.backendModule ?? '@treecrdt/postgres-napi', + "backendModule", + opts.backendModule ?? "@treecrdt/postgres-napi" ); - const postgresUrl = ensureNonEmptyString('postgresUrl', opts.postgresUrl); + const postgresUrl = ensureNonEmptyString("postgresUrl", opts.postgresUrl); + const maxCodewords = + opts.maxCodewords == null ? undefined : Number(opts.maxCodewords); + const directSendThreshold = + opts.directSendThreshold == null ? undefined : Number(opts.directSendThreshold); const idleCloseMs = Number(opts.idleCloseMs ?? 30_000); const maxPayloadBytes = Number(opts.maxPayloadBytes ?? 10 * 1024 * 1024); const authToken = - typeof opts.authToken === 'string' && opts.authToken.trim().length > 0 - ? opts.authToken.trim() - : undefined; + typeof opts.authToken === "string" && opts.authToken.trim().length > 0 ? opts.authToken.trim() : undefined; const authCapabilityIssuerPublicKeys = (opts.authCapabilityIssuerPublicKeys ?? []).filter( - (value): value is Uint8Array => value instanceof Uint8Array && value.length > 0, + (value): value is Uint8Array => value instanceof Uint8Array && value.length > 0 ); const docIdPattern = parseDocIdRegex(opts.docIdPattern); const allowDocCreate = opts.allowDocCreate ?? true; const enablePgNotify = opts.enablePgNotify ?? true; - const pgNotifyChannel = ensurePostgresChannelName( - opts.pgNotifyChannel ?? 'treecrdt_sync_doc_updates', - ); + const pgNotifyChannel = ensurePostgresChannelName(opts.pgNotifyChannel ?? "treecrdt_sync_doc_updates"); const rateLimitMaxUpgrades = Number(opts.rateLimitMaxUpgrades ?? 0); const rateLimitWindowMs = Number(opts.rateLimitWindowMs ?? 60_000); - const packageName = opts.packageName?.trim() || '@treecrdt/sync-server-postgres-node'; + const packageName = opts.packageName?.trim() || "@treecrdt/sync-server-postgres-node"; const packageVersion = opts.packageVersion?.trim() || undefined; const gitSha = opts.gitSha?.trim() || undefined; const gitDirty = Boolean(opts.gitDirty); @@ -625,8 +621,13 @@ export async function startSyncServer(opts: SyncServerOptions): Promise 0 ? createCapabilityTokenAuthHook(authCapabilityIssuerPublicKeys) : authToken - ? createStaticTokenAuthHook(authToken) - : undefined; + ? createStaticTokenAuthHook(authToken) + : undefined; const builtInAuthorizeHook = combineUpgradeHooks( docIdPattern ? createDocIdPatternHook(docIdPattern) : undefined, - !allowDocCreate ? createKnownDocHook((docId) => docUpdateBus!.hasDoc(docId)) : undefined, + !allowDocCreate ? createKnownDocHook((docId) => docUpdateBus!.hasDoc(docId)) : undefined ); const hooks: WebSocketSyncServerHooks = { onRateLimit: combineUpgradeHooks( - rateLimitMaxUpgrades > 0 - ? createIpRateLimitHook(rateLimitMaxUpgrades, rateLimitWindowMs) - : undefined, - opts.hooks?.onRateLimit, + rateLimitMaxUpgrades > 0 ? createIpRateLimitHook(rateLimitMaxUpgrades, rateLimitWindowMs) : undefined, + opts.hooks?.onRateLimit ), onAuthenticate: combineUpgradeHooks(builtInAuthHook, opts.hooks?.onAuthenticate), onAuthorize: combineUpgradeHooks(builtInAuthorizeHook, opts.hooks?.onAuthorize), @@ -730,13 +731,13 @@ export async function startSyncServer(opts: SyncServerOptions): Promise { let ready = true; - let readyDetail = 'ok'; + let readyDetail = "ok"; try { await readinessProbe!.check(); } catch (error) { @@ -752,7 +753,7 @@ export async function startSyncServer(opts: SyncServerOptions): Promise; + serverProcess: "child-process" | "in-process" | "remote"; connect: (docId: string) => Promise; + seedOps?: (docId: string, ops: Operation[]) => Promise; + waitForOpCount?: ( + docId: string, + filter: Filter, + expectedCount: number, + opts?: { timeoutMs?: number } + ) => Promise; + clearHelloTrace?: (docId: string) => void; + takeHelloTrace?: (docId: string) => HelloTraceProfile | undefined; close: () => Promise; }; +type TransportDirectionProfile = { + messages: number; + bytes: number; + codewords: number; + ops: number; + byCase: Record; +}; + +type TransportProfile = { + sent: TransportDirectionProfile; + received: TransportDirectionProfile; + events: TransportProfileEvent[]; +}; + +type TransportProfileEvent = { + direction: "sent" | "received"; + case: string; + atMs: number; + bytes: number; + codewords: number; + ops: number; +}; + +type HelloTraceRecord = { + type: "sync-hello-trace"; + docId: string; + stage: string; + ms: number; +} & Record; + +type HelloTraceProfileEvent = { + stage: string; + atMs: number; + deltaMs: number; + meta?: Record; +}; + +type HelloTraceProfile = { + totalMs: number; + events: HelloTraceProfileEvent[]; +}; + +type HelloTraceStageSummary = Record< + string, + { + count: number; + medianAtMs: number; + p95AtMs: number; + medianDeltaMs: number; + p95DeltaMs: number; + minDeltaMs: number; + maxDeltaMs: number; + } +>; + +type HelloTraceStore = { + clear: (docId: string) => void; + take: (docId: string) => HelloTraceProfile | undefined; + dispose?: () => void; +}; + const SYNC_BENCH_CONFIG: ReadonlyArray = [ [100, 10], [1_000, 5], @@ -105,7 +181,24 @@ const ALL_WORKLOADS: readonly SyncBenchWorkload[] = [ ...ALL_SYNC_BENCH_WORKLOADS, ...DEFAULT_SYNC_BENCH_ROOT_CHILDREN_WORKLOADS, ]; -const SERVER_READY_TIMEOUT_MS = 10_000; +const SYNC_SERVER_START_TIMEOUT_MS = Math.max(1_000, envInt("SYNC_BENCH_SERVER_START_TIMEOUT_MS") ?? 10_000); +const SERVER_READY_TIMEOUT_MS = Math.max(1_000, envInt("SYNC_BENCH_SERVER_READY_TIMEOUT_MS") ?? 10_000); +const SERVER_SEED_READY_TIMEOUT_MS = Math.max( + SERVER_READY_TIMEOUT_MS, + envInt("SYNC_BENCH_SEED_READY_TIMEOUT_MS") ?? 60_000 +); +const SYNC_BENCH_SERVER_MAX_CODEWORDS = Math.max( + SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + envInt("SYNC_BENCH_SERVER_MAX_CODEWORDS") ?? 1_000_000 +); +const SYNC_BENCH_SEED_MAX_CODEWORDS = Math.max( + SYNC_BENCH_SERVER_MAX_CODEWORDS, + envInt("SYNC_BENCH_SEED_MAX_CODEWORDS") ?? SYNC_BENCH_SERVER_MAX_CODEWORDS +); +const SYNC_BENCH_DIRECT_SEND_THRESHOLD = Math.max( + 0, + envInt("SYNC_BENCH_DIRECT_SEND_THRESHOLD") ?? 0 +); const SERVER_READY_POLL_MS = 100; function envInt(name: string): number | undefined { @@ -115,14 +208,34 @@ function envInt(name: string): number | undefined { return Number.isFinite(n) ? n : undefined; } -function parseConfigFromArgv(argv: string[]): Array | null { +function recommendedIterationsForCustomSize(size: number): number { + if (size <= 100) return 10; + if (size <= 1_000) return 7; + if (size <= 10_000) return 5; + return 3; +} + +function parseIterationsOverride(argv: string[]): number | undefined { + return parseOptionalPositiveIntFlag(argv, "--iterations", ["SYNC_BENCH_ITERATIONS", "BENCH_ITERATIONS"]); +} + +function parseWarmupIterationsOverride(argv: string[]): number | undefined { + return parseOptionalNonNegativeIntFlag(argv, "--warmup", ["SYNC_BENCH_WARMUP", "BENCH_WARMUP"]); +} + +function resolveWarmupIterations(iterations: number, explicitWarmupIterations?: number): number { + if (explicitWarmupIterations != null) return explicitWarmupIterations; + return iterations > 1 ? 1 : 0; +} + +function parseConfigFromArgv(argv: string[], iterationsOverride?: number): Array | null { let customConfig: Array | null = null; - const defaultIterations = Math.max(1, envInt("BENCH_ITERATIONS") ?? 1); for (const arg of argv) { if (arg.startsWith("--count=")) { const val = arg.slice("--count=".length).trim(); const count = val ? Number(val) : 500; - customConfig = [[Number.isFinite(count) && count > 0 ? count : 500, defaultIterations]]; + const normalizedCount = Number.isFinite(count) && count > 0 ? count : 500; + customConfig = [[normalizedCount, iterationsOverride ?? recommendedIterationsForCustomSize(normalizedCount)]]; break; } if (arg.startsWith("--counts=")) { @@ -137,7 +250,10 @@ function parseConfigFromArgv(argv: string[]): Array | null { return Number.isFinite(n) && n > 0 ? n : null; }) .filter((n): n is number => n != null) - .map((c) => [c, defaultIterations] as ConfigEntry); + .map( + (c) => + [c, iterationsOverride ?? recommendedIterationsForCustomSize(c)] as ConfigEntry + ); if (parsed.length > 0) customConfig = parsed; break; } @@ -161,6 +277,38 @@ function parsePositiveIntFlag(argv: string[], flag: string, envName: string, fal return value; } +function parseOptionalPositiveIntFlag( + argv: string[], + flag: string, + envNames: readonly string[] +): number | undefined { + const raw = + parseFlagValue(argv, flag) ?? + envNames.map((name) => process.env[name]).find((value) => value != null && value !== ""); + if (!raw) return undefined; + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`invalid ${flag} value "${raw}", expected a positive integer`); + } + return value; +} + +function parseOptionalNonNegativeIntFlag( + argv: string[], + flag: string, + envNames: readonly string[] +): number | undefined { + const raw = + parseFlagValue(argv, flag) ?? + envNames.map((name) => process.env[name]).find((value) => value != null && value !== ""); + if (!raw) return undefined; + const value = Number(raw); + if (!Number.isInteger(value) || value < 0) { + throw new Error(`invalid ${flag} value "${raw}", expected a non-negative integer`); + } + return value; +} + function parseBooleanFlag(argv: string[], flag: string, envName: string): boolean { const envRaw = process.env[envName]; if (argv.includes(flag)) return true; @@ -269,6 +417,300 @@ function parseProfileBackend(argv: string[]): boolean { return parseBooleanFlag(argv, "--profile-backend", "SYNC_BENCH_PROFILE_BACKEND"); } +function parseProfileTransport(argv: string[]): boolean { + return parseBooleanFlag(argv, "--profile-transport", "SYNC_BENCH_PROFILE_TRANSPORT"); +} + +function parseProfileHello(argv: string[]): boolean { + return parseBooleanFlag(argv, "--profile-hello", "SYNC_BENCH_PROFILE_HELLO"); +} + +function parseDirectSendThreshold(argv: string[]): number { + return parseOptionalNonNegativeIntFlag(argv, "--direct-send-threshold", [ + "SYNC_BENCH_DIRECT_SEND_THRESHOLD", + ]) ?? SYNC_BENCH_DIRECT_SEND_THRESHOLD; +} + +function createEmptyTransportDirectionProfile(): TransportDirectionProfile { + return { + messages: 0, + bytes: 0, + codewords: 0, + ops: 0, + byCase: {}, + }; +} + +function createEmptyTransportProfile(): TransportProfile { + return { + sent: createEmptyTransportDirectionProfile(), + received: createEmptyTransportDirectionProfile(), + events: [], + }; +} + +function snapshotTransportProfile(profile: TransportProfile): TransportProfile { + return { + sent: { + ...profile.sent, + byCase: { ...profile.sent.byCase }, + }, + received: { + ...profile.received, + byCase: { ...profile.received.byCase }, + }, + events: profile.events.map((event) => ({ ...event })), + }; +} + +function recordTransportMessage( + profile: TransportProfile, + directionName: "sent" | "received", + direction: TransportDirectionProfile, + message: any +): void { + direction.messages += 1; + let bytes = 0; + try { + bytes = treecrdtSyncV0ProtobufCodec.encode(message).byteLength; + direction.bytes += bytes; + } catch { + // best-effort; message counts are still useful if encoding fails + } + + const payloadCase = + typeof message?.payload?.case === "string" ? message.payload.case : "unknown"; + direction.byCase[payloadCase] = (direction.byCase[payloadCase] ?? 0) + 1; + + if (payloadCase === "ribltCodewords") { + const codewords = message?.payload?.value?.codewords; + if (Array.isArray(codewords)) { + direction.codewords += codewords.length; + } + } + + if (payloadCase === "opsBatch") { + const ops = message?.payload?.value?.ops; + if (Array.isArray(ops)) { + direction.ops += ops.length; + } + } + + if (profile.events.length < 128) { + profile.events.push({ + direction: directionName, + case: payloadCase, + atMs: performance.now() - transportProfileStartTimes.get(profile)!, + bytes, + codewords: + payloadCase === "ribltCodewords" && Array.isArray(message?.payload?.value?.codewords) + ? message.payload.value.codewords.length + : 0, + ops: + payloadCase === "opsBatch" && Array.isArray(message?.payload?.value?.ops) + ? message.payload.value.ops.length + : 0, + }); + } +} + +const transportProfileStartTimes = new WeakMap(); + +function createProfiledSyncTransport( + transport: DuplexTransport +): { + transport: DuplexTransport; + snapshot: () => TransportProfile; +} { + const profile = createEmptyTransportProfile(); + transportProfileStartTimes.set(profile, performance.now()); + return { + transport: { + send: async (message) => { + recordTransportMessage(profile, "sent", profile.sent, message); + await transport.send(message); + }, + onMessage: (handler) => + transport.onMessage((message) => { + recordTransportMessage(profile, "received", profile.received, message); + handler(message); + }), + }, + snapshot: () => snapshotTransportProfile(profile), + }; +} + +const HELLO_TRACE_SINK_KEY = "__TREECRDT_SYNC_HELLO_TRACE_SINK__"; + +function normalizeHelloTraceRecord(value: unknown): HelloTraceRecord | null { + if (!value || typeof value !== "object") return null; + const record = value as Record; + if (record.type !== "sync-hello-trace") return null; + if (typeof record.docId !== "string" || typeof record.stage !== "string" || typeof record.ms !== "number") { + return null; + } + return record as HelloTraceRecord; +} + +function buildHelloTraceProfile(records: HelloTraceRecord[]): HelloTraceProfile | undefined { + if (records.length === 0) return undefined; + const sorted = records.slice().sort((a, b) => a.ms - b.ms); + let previousMs = 0; + return { + totalMs: sorted[sorted.length - 1]!.ms, + events: sorted.map((record, index) => { + const { type: _type, docId: _docId, stage, ms, ...meta } = record; + const deltaMs = index === 0 ? ms : ms - previousMs; + previousMs = ms; + return { + stage, + atMs: ms, + deltaMs, + meta: Object.keys(meta).length > 0 ? meta : undefined, + }; + }), + }; +} + +function summarizeHelloTraceProfiles( + profiles: HelloTraceProfile[] +): HelloTraceStageSummary | undefined { + if (profiles.length === 0) return undefined; + const buckets = new Map(); + for (const profile of profiles) { + for (const event of profile.events) { + const bucket = buckets.get(event.stage) ?? { atMs: [], deltaMs: [] }; + bucket.atMs.push(event.atMs); + bucket.deltaMs.push(event.deltaMs); + buckets.set(event.stage, bucket); + } + } + + return Object.fromEntries( + Array.from(buckets.entries(), ([stage, values]) => [ + stage, + { + count: values.atMs.length, + medianAtMs: quantile(values.atMs, 0.5), + p95AtMs: quantile(values.atMs, 0.95), + medianDeltaMs: quantile(values.deltaMs, 0.5), + p95DeltaMs: quantile(values.deltaMs, 0.95), + minDeltaMs: Math.min(...values.deltaMs), + maxDeltaMs: Math.max(...values.deltaMs), + }, + ]) + ); +} + +function createChunkLineParser(onLine: (line: string) => void): { + push: (chunk: Buffer | string) => void; + flush: () => void; +} { + let buffer = ""; + const handleLines = () => { + while (true) { + const newlineIndex = buffer.indexOf("\n"); + if (newlineIndex === -1) return; + const line = buffer.slice(0, newlineIndex).trim(); + buffer = buffer.slice(newlineIndex + 1); + if (line) onLine(line); + } + }; + + return { + push: (chunk) => { + buffer += chunk.toString(); + handleLines(); + }, + flush: () => { + const line = buffer.trim(); + buffer = ""; + if (line) onLine(line); + }, + }; +} + +function createChildProcessHelloTraceStore(): HelloTraceStore & { + pushStdout: (chunk: Buffer | string) => void; + pushStderr: (chunk: Buffer | string) => void; +} { + const traces = new Map(); + const remember = (record: HelloTraceRecord) => { + const current = traces.get(record.docId); + if (current) { + current.push(record); + return; + } + traces.set(record.docId, [record]); + }; + const onLine = (line: string) => { + if (!line.includes("\"sync-hello-trace\"")) return; + try { + const parsed = normalizeHelloTraceRecord(JSON.parse(line)); + if (parsed) remember(parsed); + } catch { + // ignore non-JSON or partial debug output + } + }; + const stdoutParser = createChunkLineParser(onLine); + const stderrParser = createChunkLineParser(onLine); + + return { + pushStdout: stdoutParser.push, + pushStderr: stderrParser.push, + clear: (docId) => { + traces.delete(docId); + }, + take: (docId) => { + const records = traces.get(docId) ?? []; + traces.delete(docId); + return buildHelloTraceProfile(records); + }, + dispose: () => { + stdoutParser.flush(); + stderrParser.flush(); + traces.clear(); + }, + }; +} + +function createProcessHelloTraceStore(): HelloTraceStore { + const traces = new Map(); + const root = globalThis as Record; + const previousSink = root[HELLO_TRACE_SINK_KEY]; + const nextSink = (record: HelloTraceRecord) => { + if (typeof previousSink === "function") { + (previousSink as (record: HelloTraceRecord) => void)(record); + } + const current = traces.get(record.docId); + if (current) { + current.push(record); + return; + } + traces.set(record.docId, [record]); + }; + root[HELLO_TRACE_SINK_KEY] = nextSink; + + return { + clear: (docId) => { + traces.delete(docId); + }, + take: (docId) => { + const records = traces.get(docId) ?? []; + traces.delete(docId); + return buildHelloTraceProfile(records); + }, + dispose: () => { + if (previousSink === undefined) { + delete root[HELLO_TRACE_SINK_KEY]; + } else { + root[HELLO_TRACE_SINK_KEY] = previousSink; + } + traces.clear(); + }, + }; +} + function normalizeSyncServerUrl(raw: string, docId: string): URL { let input = raw.trim(); if (input.length === 0) throw new Error("Sync server URL is empty"); @@ -482,7 +924,9 @@ async function connectToSyncServer( async function createLocalPostgresSyncServerTarget( repoRoot: string, postgresUrl: string, - profileBackend: boolean + profileBackend: boolean, + profileHello: boolean, + directSendThreshold: number ): Promise { const port = await findFreePort(); const backendModule = profileBackend @@ -501,25 +945,173 @@ async function createLocalPostgresSyncServerTarget( "index.js" ); - const server = await startSyncServer({ - host: "127.0.0.1", - port, - postgresUrl, + const waitForOpCount = await createDirectPostgresOpCountWaiter( + backendModule, + postgresUrl + ); + const backendFactory = await loadPostgresSyncBackendFactory( backendModule, - allowDocCreate: true, - enablePgNotify: false, + postgresUrl + ); + const seedOps = async (docId: string, ops: Operation[]) => { + if (ops.length === 0) return; + const backend = await backendFactory.open(docId); + await backend.applyOps(ops); + }; + + if (profileBackend) { + const server = await startSyncServer({ + host: "127.0.0.1", + port, + postgresUrl, + backendModule, + maxCodewords: SYNC_BENCH_SERVER_MAX_CODEWORDS, + ...(directSendThreshold > 0 ? { directSendThreshold } : {}), + allowDocCreate: true, + enablePgNotify: false, + }); + + return { + id: "local-postgres-sync-server", + serverProcess: "in-process", + connect: async (docId) => + await connectToSyncServer(`ws://127.0.0.1:${server.port}`, docId), + seedOps, + waitForOpCount, + close: async () => { + await server.close(); + }, + }; + } + + const cliPath = path.join( + repoRoot, + "packages", + "sync", + "server", + "postgres-node", + "dist", + "cli.js" + ); + let recentOutput = ""; + const helloTraceStore = profileHello ? createChildProcessHelloTraceStore() : undefined; + const child = spawn(process.execPath, [cliPath], { + cwd: repoRoot, + env: { + ...process.env, + HOST: "127.0.0.1", + PORT: String(port), + TREECRDT_POSTGRES_URL: postgresUrl, + TREECRDT_POSTGRES_BACKEND_MODULE: backendModule, + TREECRDT_SYNC_MAX_CODEWORDS: String(SYNC_BENCH_SERVER_MAX_CODEWORDS), + TREECRDT_SYNC_DIRECT_SEND_THRESHOLD: String(directSendThreshold), + TREECRDT_ALLOW_DOC_CREATE: "true", + TREECRDT_PG_NOTIFY_ENABLED: "false", + ...(profileHello ? { TREECRDT_SYNC_TRACE_HELLO: "1" } : {}), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + const rememberOutput = (chunk: Buffer | string) => { + recentOutput += chunk.toString(); + if (recentOutput.length > 8_000) { + recentOutput = recentOutput.slice(-8_000); + } + }; + child.stdout?.on("data", (chunk) => { + rememberOutput(chunk); + helloTraceStore?.pushStdout(chunk); + }); + child.stderr?.on("data", (chunk) => { + rememberOutput(chunk); + helloTraceStore?.pushStderr(chunk); }); + await waitForSyncServerReady( + `http://127.0.0.1:${port}/health`, + child, + () => recentOutput + ); + return { id: "local-postgres-sync-server", + serverProcess: "child-process", connect: async (docId) => - await connectToSyncServer(`ws://127.0.0.1:${server.port}`, docId), + await connectToSyncServer(`ws://127.0.0.1:${port}`, docId), + seedOps, + waitForOpCount, + clearHelloTrace: helloTraceStore ? (docId) => helloTraceStore.clear(docId) : undefined, + takeHelloTrace: helloTraceStore ? (docId) => helloTraceStore.take(docId) : undefined, close: async () => { - await server.close(); + helloTraceStore?.dispose?.(); + if (child.exitCode == null && child.signalCode == null) { + child.kill("SIGTERM"); + await waitForProcessExit(child, 5_000); + } }, }; } +async function createDirectPostgresOpCountWaiter( + backendModule: string, + postgresUrl: string +): Promise< + ( + docId: string, + filter: Filter, + expectedCount: number, + opts?: { timeoutMs?: number } + ) => Promise +> { + const factory = await loadPostgresSyncBackendFactory(backendModule, postgresUrl); + return async (docId: string, filter: Filter, expectedCount: number, opts?: { timeoutMs?: number }) => { + const timeoutMs = Math.max(1_000, opts?.timeoutMs ?? SERVER_READY_TIMEOUT_MS); + const deadline = Date.now() + timeoutMs; + let lastCount = -1; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const backend = await factory.open(docId); + const refs = await backend.listOpRefs(filter); + lastCount = refs.length; + if (refs.length === expectedCount) { + return; + } + } catch (error) { + lastError = error; + } + await sleep(SERVER_READY_POLL_MS); + } + throw new Error( + `timed out waiting for server doc ${docId} to reach ${expectedCount} ops within ${timeoutMs}ms` + + (lastCount >= 0 ? ` (last count=${lastCount})` : "") + + (lastError ? `: ${String(lastError)}` : "") + ); + }; +} + +async function loadPostgresSyncBackendFactory( + backendModule: string, + postgresUrl: string +): Promise<{ + open: (docId: string) => Promise<{ + applyOps: (ops: Operation[]) => Promise; + listOpRefs: (filter: Filter) => Promise; + }>; +}> { + const mod = (await import(pathToFileURL(backendModule).href)) as { + createPostgresNapiSyncBackendFactory?: (url: string) => { + open: (docId: string) => Promise<{ + applyOps: (ops: Operation[]) => Promise; + listOpRefs: (filter: Filter) => Promise; + }>; + }; + }; + if (typeof mod.createPostgresNapiSyncBackendFactory !== "function") { + throw new Error(`backend module "${backendModule}" does not export createPostgresNapiSyncBackendFactory(url)`); + } + return mod.createPostgresNapiSyncBackendFactory(postgresUrl); +} + async function findFreePort(): Promise { return await new Promise((resolve, reject) => { const server = net.createServer(); @@ -548,16 +1140,69 @@ function createRemoteSyncServerTarget( ): SyncBenchTargetRuntime { return { id: "remote-sync-server", + serverProcess: "remote", connect: async (docId) => await connectToSyncServer(baseUrl, docId), close: async () => {}, }; } +async function waitForProcessExit( + child: ReturnType, + timeoutMs: number +): Promise { + if (child.exitCode != null || child.signalCode != null) return; + await new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(() => { + if (child.exitCode == null && child.signalCode == null) { + child.kill("SIGKILL"); + } + finish(); + }, timeoutMs); + child.once("exit", finish); + }); +} + +async function waitForSyncServerReady( + healthUrl: string, + child: ReturnType, + getRecentOutput: () => string +): Promise { + const deadline = Date.now() + SYNC_SERVER_START_TIMEOUT_MS; + while (Date.now() < deadline) { + if (child.exitCode != null || child.signalCode != null) { + const recentOutput = getRecentOutput(); + throw new Error( + `local sync server exited before becoming ready (${healthUrl})${recentOutput ? `\n${recentOutput}` : ""}` + ); + } + try { + const response = await fetch(healthUrl); + if (response.ok) return; + } catch { + // keep polling until timeout + } + await sleep(SERVER_READY_POLL_MS); + } + const recentOutput = getRecentOutput(); + throw new Error( + `timed out waiting for local sync server readiness (${healthUrl})${recentOutput ? `\n${recentOutput}` : ""}` + ); +} + async function prepareTargetRuntimes( repoRoot: string, argv: string[], targets: SyncBenchTargetId[], - profileBackend: boolean + profileBackend: boolean, + profileHello: boolean, + directSendThreshold: number ): Promise, SyncBenchTargetRuntime>> { const runtimes = new Map< Exclude, @@ -574,7 +1219,13 @@ async function prepareTargetRuntimes( } runtimes.set( "local-postgres-sync-server", - await createLocalPostgresSyncServerTarget(repoRoot, postgresUrl, profileBackend) + await createLocalPostgresSyncServerTarget( + repoRoot, + postgresUrl, + profileBackend, + profileHello, + directSendThreshold + ) ); } @@ -599,22 +1250,45 @@ async function closeTargetRuntimes( await Promise.allSettled(Array.from(runtimes.values(), (runtime) => runtime.close())); } +function getRuntimeHelloTraceStore( + runtime: SyncBenchTargetRuntime, + profileHello: boolean +): HelloTraceStore | undefined { + if (!profileHello) return undefined; + if (runtime.clearHelloTrace && runtime.takeHelloTrace) { + return { + clear: runtime.clearHelloTrace, + take: runtime.takeHelloTrace, + }; + } + if (runtime.serverProcess === "in-process") { + return createProcessHelloTraceStore(); + } + return undefined; +} + async function syncBackendThroughServer( runtime: SyncBenchTargetRuntime, docId: string, backend: FlushableSyncBackend, - filter: Filter + filter: Filter, + opts: { + maxCodewords?: number; + codewordsPerMessage?: number; + directSendThreshold?: number; + } = {} ): Promise { const peer = new SyncPeer(backend, { - maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + maxCodewords: opts.maxCodewords ?? SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + directSendThreshold: opts.directSendThreshold ?? 0, }); const connection = await runtime.connect(docId); const detach = peer.attach(connection.transport); try { await peer.syncOnce(connection.transport, filter, { - maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, - codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + maxCodewords: opts.maxCodewords ?? SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + codewordsPerMessage: opts.codewordsPerMessage ?? SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, }); await backend.flush(); } finally { @@ -626,9 +1300,10 @@ async function syncBackendThroughServer( async function seedServerState( runtime: SyncBenchTargetRuntime, docId: string, - ops: Operation[] -): Promise { - if (ops.length === 0) return; + ops: Operation[], + filter: Filter +): Promise { + if (ops.length === 0) return 0; const seedDb = await openDb({ storage: "memory", docId }); try { @@ -638,7 +1313,38 @@ async function seedServerState( docId, initialMaxLamport: maxLamport(ops), }); - await syncBackendThroughServer(runtime, docId, seedBackend, { all: {} }); + const expectedFilterCount = (await seedBackend.listOpRefs(filter)).length; + if (runtime.seedOps) { + await runtime.seedOps(docId, ops); + if (runtime.waitForOpCount) { + await runtime.waitForOpCount(docId, { all: {} }, ops.length, { + timeoutMs: SERVER_SEED_READY_TIMEOUT_MS, + }); + } + return expectedFilterCount; + } + const peer = new SyncPeer(seedBackend, { + maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, + directSendThreshold: 0, + }); + const connection = await runtime.connect(docId); + const detach = peer.attach(connection.transport); + try { + await peer.syncOnce(connection.transport, { all: {} }, { + maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, + codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + }); + await seedBackend.flush(); + if (runtime.waitForOpCount) { + await runtime.waitForOpCount(docId, { all: {} }, ops.length, { + timeoutMs: SERVER_SEED_READY_TIMEOUT_MS, + }); + } + } finally { + detach(); + await connection.close(); + } + return expectedFilterCount; } finally { seedDb.close(); } @@ -647,9 +1353,12 @@ async function seedServerState( async function waitForServerOpCount( runtime: SyncBenchTargetRuntime, docId: string, - expectedCount: number + expectedCount: number, + directSendThreshold: number, + timeoutMs = SERVER_READY_TIMEOUT_MS ): Promise { - const deadline = Date.now() + SERVER_READY_TIMEOUT_MS; + const deadline = Date.now() + timeoutMs; + let lastCount = -1; while (true) { const verifierDb = await openDb({ storage: "memory", docId }); try { @@ -658,8 +1367,12 @@ async function waitForServerOpCount( docId, initialMaxLamport: 0, }); - await syncBackendThroughServer(runtime, docId, verifierBackend, { all: {} }); - if (countOps(verifierDb) === expectedCount) { + await syncBackendThroughServer(runtime, docId, verifierBackend, { all: {} }, { + maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, + directSendThreshold, + }); + lastCount = countOps(verifierDb); + if (lastCount === expectedCount) { return; } } finally { @@ -668,7 +1381,8 @@ async function waitForServerOpCount( if (Date.now() >= deadline) { throw new Error( - `timed out waiting for server doc ${docId} to reach ${expectedCount} ops` + `timed out waiting for server doc ${docId} to reach ${expectedCount} ops within ${timeoutMs}ms` + + (lastCount >= 0 ? ` (last count=${lastCount})` : "") ); } await sleep(SERVER_READY_POLL_MS); @@ -709,10 +1423,14 @@ async function runBenchOnceDirect( { storage, workload, size }: BenchCase, bench: ReturnType, includeFirstView: boolean, - profileBackend: boolean + profileBackend: boolean, + profileTransport: boolean, + profileHello: boolean, + directSendThreshold: number ): Promise { const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runId}`; + const helloTraceStore = profileHello ? createProcessHelloTraceStore() : undefined; const clientA = await openClientDbForRun( repoRoot, storage, @@ -749,17 +1467,29 @@ async function runBenchOnceDirect( profileLabel: profileBackend ? "direct-client-b" : undefined, }); - const { peerA: pa, transportA: ta, detach } = createInMemoryConnectedPeers({ - backendA, - backendB, - codec: treecrdtSyncV0ProtobufCodec, - peerOptions: { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS }, + const [wireA, wireB] = createInMemoryDuplex(); + const baseTransportA = wrapDuplexTransportWithCodec(wireA, treecrdtSyncV0ProtobufCodec); + const transportAProfile = profileTransport + ? createProfiledSyncTransport(baseTransportA) + : undefined; + const transportA = transportAProfile?.transport ?? baseTransportA; + const transportB = wrapDuplexTransportWithCodec(wireB, treecrdtSyncV0ProtobufCodec); + const pa = new SyncPeer(backendA, { + maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + directSendThreshold, + }); + const pb = new SyncPeer(backendB, { + maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + directSendThreshold, }); + const detachA = pa.attach(transportA); + const detachB = pb.attach(transportB); try { if (profileBackend) resetBackendProfiler(docId); + helloTraceStore?.clear(docId); const start = performance.now(); - await pa.syncOnce(ta, bench.filter as Filter, { + await pa.syncOnce(transportA, bench.filter as Filter, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, }); @@ -790,11 +1520,15 @@ async function runBenchOnceDirect( syncMs: syncedAt - start, firstViewReadMs, backendProfile: profileBackend ? takeBackendProfilerSnapshot(docId) : undefined, + transportProfile: profileTransport ? transportAProfile?.snapshot() : undefined, + helloProfile: helloTraceStore?.take(docId), }; } finally { - detach(); + detachA(); + detachB(); } } finally { + helloTraceStore?.dispose?.(); await Promise.all([clientA.cleanup(), clientB.cleanup()]); } } @@ -805,10 +1539,14 @@ async function runBenchOnceViaServer( { storage, workload, size }: BenchCase, bench: ReturnType, includeFirstView: boolean, - profileBackend: boolean + profileBackend: boolean, + profileTransport: boolean, + profileHello: boolean, + directSendThreshold: number ): Promise { const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runtime.id}-${runId}`; + const helloTraceStore = getRuntimeHelloTraceStore(runtime, profileHello); const client = await openClientDbForRun( repoRoot, storage, @@ -820,8 +1558,23 @@ async function runBenchOnceViaServer( try { await appendInitialOps(client.db, bench.opsA); - await seedServerState(runtime, docId, bench.opsB); - await waitForServerOpCount(runtime, docId, bench.opsB.length); + const expectedFilterCount = await seedServerState( + runtime, + docId, + bench.opsB, + bench.filter as Filter + ); + if (runtime.waitForOpCount) { + await runtime.waitForOpCount( + docId, + bench.filter as Filter, + expectedFilterCount, + { timeoutMs: SERVER_READY_TIMEOUT_MS } + ); + } else { + await waitForServerOpCount(runtime, docId, bench.opsB.length, directSendThreshold, SERVER_SEED_READY_TIMEOUT_MS); + } + helloTraceStore?.clear(docId); const clientBackend = await makeBackend({ db: client.db, @@ -832,14 +1585,19 @@ async function runBenchOnceViaServer( const peer = new SyncPeer(clientBackend, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + directSendThreshold, }); const connection = await runtime.connect(docId); - const detach = peer.attach(connection.transport); + const transportProfile = profileTransport + ? createProfiledSyncTransport(connection.transport) + : undefined; + const transport = transportProfile?.transport ?? connection.transport; + const detach = peer.attach(transport); try { if (profileBackend) resetBackendProfiler(docId); const start = performance.now(); - await peer.syncOnce(connection.transport, bench.filter as Filter, { + await peer.syncOnce(transport, bench.filter as Filter, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, }); @@ -866,12 +1624,15 @@ async function runBenchOnceViaServer( syncMs: syncedAt - start, firstViewReadMs, backendProfile: profileBackend ? takeBackendProfilerSnapshot(docId) : undefined, + transportProfile: profileTransport ? transportProfile?.snapshot() : undefined, + helloProfile: helloTraceStore?.take(docId), }; } finally { detach(); await connection.close(); } } finally { + helloTraceStore?.dispose?.(); await client.cleanup(); } } @@ -881,14 +1642,17 @@ async function runBenchCase( benchCase: BenchCase, runtimes: Map, SyncBenchTargetRuntime>, includeFirstView: boolean, - profileBackend: boolean + profileBackend: boolean, + profileTransport: boolean, + profileHello: boolean, + directSendThreshold: number ): Promise { const bench = buildSyncBenchCase({ workload: benchCase.workload, size: benchCase.size, fanout: benchCase.fanout, }); - const { iterations } = benchCase; + const { iterations, warmupIterations } = benchCase; const runtime = benchCase.target === "direct" ? null : runtimes.get(benchCase.target); @@ -900,12 +1664,58 @@ async function runBenchCase( throw new Error(`sync bench workload ${bench.name} does not support --first-view`); } + for (let i = 0; i < warmupIterations; i += 1) { + if (runtime) { + await runBenchOnceViaServer( + repoRoot, + runtime, + benchCase, + bench, + includeFirstView, + profileBackend, + profileTransport, + profileHello, + directSendThreshold + ); + } else { + await runBenchOnceDirect( + repoRoot, + benchCase, + bench, + includeFirstView, + profileBackend, + profileTransport, + profileHello, + directSendThreshold + ); + } + } + const samples: SyncBenchSample[] = []; for (let i = 0; i < iterations; i += 1) { samples.push( runtime - ? await runBenchOnceViaServer(repoRoot, runtime, benchCase, bench, includeFirstView, profileBackend) - : await runBenchOnceDirect(repoRoot, benchCase, bench, includeFirstView, profileBackend) + ? await runBenchOnceViaServer( + repoRoot, + runtime, + benchCase, + bench, + includeFirstView, + profileBackend, + profileTransport, + profileHello, + directSendThreshold + ) + : await runBenchOnceDirect( + repoRoot, + benchCase, + bench, + includeFirstView, + profileBackend, + profileTransport, + profileHello, + directSendThreshold + ) ); } @@ -915,6 +1725,12 @@ async function runBenchCase( const backendProfiles = samples .map((sample) => sample.backendProfile) .filter((sample): sample is NonNullable => sample != null); + const transportProfiles = samples + .map((sample) => sample.transportProfile) + .filter((sample): sample is NonNullable => sample != null); + const helloProfiles = samples + .map((sample) => sample.helloProfile) + .filter((sample): sample is NonNullable => sample != null); const durationMs = iterations > 1 ? quantile(totalSamplesMs, 0.5) : totalSamplesMs[0] ?? 0; const opsPerSec = durationMs > 0 ? (bench.totalOps / durationMs) * 1000 : Infinity; @@ -937,12 +1753,23 @@ async function runBenchCase( : benchCase.target === "remote-sync-server" ? "remote" : "none", + serverProcess: + runtime?.serverProcess ?? (benchCase.target === "direct" ? "none" : "unknown"), measurement: includeFirstView ? "time-to-first-view" : "sync-only", backendProfile: profileBackend ? backendProfiles.at(-1) : undefined, backendProfileSamples: profileBackend && backendProfiles.length > 1 ? backendProfiles : undefined, + transportProfile: profileTransport ? transportProfiles.at(-1) : undefined, + transportProfileSamples: + profileTransport && transportProfiles.length > 1 ? transportProfiles : undefined, + helloProfile: profileHello ? helloProfiles.at(-1) : undefined, + helloProfileSamples: profileHello && helloProfiles.length > 1 ? helloProfiles : undefined, + helloStageSummary: profileHello ? summarizeHelloTraceProfiles(helloProfiles) : undefined, + helloTotalMsSamples: profileHello ? helloProfiles.map((profile) => profile.totalMs) : undefined, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, + directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, iterations: iterations > 1 ? iterations : undefined, + warmupIterations: warmupIterations > 0 ? warmupIterations : undefined, avgDurationMs: iterations > 1 ? durationMs : undefined, samplesMs: totalSamplesMs, syncSamplesMs, @@ -960,15 +1787,33 @@ async function main() { const argv = process.argv.slice(2); const repoRoot = repoRootFromImportMeta(import.meta.url, 3); - const config = parseConfigFromArgv(argv) ?? [...SYNC_BENCH_CONFIG]; - const rootConfig = [...SYNC_BENCH_ROOT_CONFIG]; + const iterationsOverride = parseIterationsOverride(argv); + const warmupIterationsOverride = parseWarmupIterationsOverride(argv); + const config = + parseConfigFromArgv(argv, iterationsOverride) ?? + SYNC_BENCH_CONFIG.map( + ([size, iterations]) => [size, iterationsOverride ?? iterations] as ConfigEntry + ); + const rootConfig = SYNC_BENCH_ROOT_CONFIG.map( + ([size, iterations]) => [size, iterationsOverride ?? iterations] as ConfigEntry + ); const targets = parseTargets(argv); const storages = parseStorages(argv); const workloads = parseWorkloads(argv); const fanout = parseFanout(argv); const includeFirstView = parseFirstView(argv); const profileBackend = parseProfileBackend(argv); - const runtimes = await prepareTargetRuntimes(repoRoot, argv, targets, profileBackend); + const profileTransport = parseProfileTransport(argv); + const profileHello = parseProfileHello(argv); + const directSendThreshold = parseDirectSendThreshold(argv); + const runtimes = await prepareTargetRuntimes( + repoRoot, + argv, + targets, + profileBackend, + profileHello, + directSendThreshold + ); try { const cases: BenchCase[] = []; @@ -978,14 +1823,31 @@ async function main() { const entries = workload === "sync-root-children-fanout10" ? rootConfig : config; for (const [size, iterations] of entries) { - cases.push({ target, storage, workload, size, iterations, fanout }); + cases.push({ + target, + storage, + workload, + size, + iterations, + warmupIterations: resolveWarmupIterations(iterations, warmupIterationsOverride), + fanout, + }); } } } } for (const benchCase of cases) { - const result = await runBenchCase(repoRoot, benchCase, runtimes, includeFirstView, profileBackend); + const result = await runBenchCase( + repoRoot, + benchCase, + runtimes, + includeFirstView, + profileBackend, + profileTransport, + profileHello, + directSendThreshold + ); const outFile = path.join( repoRoot, "benchmarks", diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index 3df1445c..74e3bb57 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -128,7 +128,11 @@ async function main() { pnpm benchmark:sync:direct -- --workloads=sync-balanced-children-payloads-cold-start --counts=10000,50000 --fanout=10 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --iterations=5 --warmup=1 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 --profile-backend + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 --profile-transport + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --profile-hello + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --direct-send-threshold=64 TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start Notes: @@ -137,7 +141,13 @@ Notes: - use wss:// for public HTTPS/TLS deployments and ws:// for local/plain HTTP servers - use --fanout=20 to model broader trees; default fanout is 10 - add --first-view to include the immediate local read after sync in the measured duration - - add --profile-backend to capture listOpRefs/getOpsByOpRefs/applyOps timing per backend + - add --iterations=N and --warmup=N to control sample count explicitly; custom --count/--counts runs now default to multiple samples instead of silently dropping to 1 + - local sync benches use a spawned child-process server by default for more realistic local vs remote comparisons + - local sync benches now use a benchmark-only direct Postgres seed step before timing, so large local runs avoid spending minutes protocol-seeding data that is not part of the measured sync + - add --profile-backend to capture listOpRefs/getOpsByOpRefs/applyOps timing per backend; on local benches this switches back to the in-process server for debug visibility + - add --profile-transport to capture sync message counts, bytes, and a small event timeline + - add --profile-hello to capture responder-side hello stage timings; local child-process runs parse server trace output, direct and in-process runs collect it in-process + - add --direct-send-threshold=N to experiment with a clean-slate shortcut that skips the RIBLT round when the requested local filter is empty and the responder has at most N scoped ops - extra args are forwarded to packages/treecrdt-sqlite-node/scripts/bench-sync.ts`); return; } From 5f058a1c9ccfe7a72c02d66d460ae9fd3ed4c6b5 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sat, 21 Mar 2026 15:12:27 +0100 Subject: [PATCH 05/58] feat(benchmark): reuse local sync fixtures across samples --- docs/BENCHMARKS.md | 2 + .../scripts/bench-sync.ts | 102 +++++++++++++++--- 2 files changed, 87 insertions(+), 17 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index e77cb75e..c2c514de 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -79,6 +79,8 @@ By default, the local sync target runs the Postgres sync server in a spawned chi Local server benchmarks now seed the Postgres backend directly before the timer starts. That keeps the measured path honest, because the actual sync to the client still goes through the real websocket server, while avoiding huge protocol-seed setup costs that are not part of the benchmark question. +For read-only local server workloads, the harness now prepares that server fixture once per benchmark case and reuses it across warmup and measured samples. That removes repeated `50k/100k` Postgres imports from the per-sample path while keeping each client run a fresh-device sync. + ### Time To First Visible Page Add `--first-view` when you want one number that includes: diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index a4c3aaa5..35e912b3 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -80,6 +80,10 @@ type SyncBenchSample = { helloProfile?: HelloTraceProfile; }; +type PreparedServerFixture = { + docId: string; +}; + type SyncBenchConnection = { transport: DuplexTransport; close: () => Promise; @@ -1350,6 +1354,52 @@ async function seedServerState( } } +function canReuseServerFixture( + runtime: SyncBenchTargetRuntime, + bench: ReturnType +): boolean { + if (runtime.id !== "local-postgres-sync-server") return false; + // Reuse is safe when every client-side starting op is already present on the + // server fixture, so samples only read from the shared doc and never mutate it. + const serverOpIds = new Set( + bench.opsB.map((op) => `${Buffer.from(op.meta.id.replica).toString("base64")}:${op.meta.id.counter}`) + ); + return bench.opsA.every((op) => + serverOpIds.has(`${Buffer.from(op.meta.id.replica).toString("base64")}:${op.meta.id.counter}`) + ); +} + +async function prepareServerFixture( + runtime: SyncBenchTargetRuntime, + bench: ReturnType, + directSendThreshold: number +): Promise { + const docId = `sqlite-node-sync-bench-${runtime.id}-fixture-${crypto.randomUUID()}`; + const expectedFilterCount = await seedServerState( + runtime, + docId, + bench.opsB, + bench.filter as Filter + ); + if (runtime.waitForOpCount) { + await runtime.waitForOpCount( + docId, + bench.filter as Filter, + expectedFilterCount, + { timeoutMs: SERVER_READY_TIMEOUT_MS } + ); + } else { + await waitForServerOpCount( + runtime, + docId, + bench.opsB.length, + directSendThreshold, + SERVER_SEED_READY_TIMEOUT_MS + ); + } + return { docId }; +} + async function waitForServerOpCount( runtime: SyncBenchTargetRuntime, docId: string, @@ -1542,10 +1592,13 @@ async function runBenchOnceViaServer( profileBackend: boolean, profileTransport: boolean, profileHello: boolean, - directSendThreshold: number + directSendThreshold: number, + preparedFixture?: PreparedServerFixture ): Promise { const runId = crypto.randomUUID(); - const docId = `sqlite-node-sync-bench-${runtime.id}-${runId}`; + const docId = + preparedFixture?.docId ?? + `sqlite-node-sync-bench-${runtime.id}-${runId}`; const helloTraceStore = getRuntimeHelloTraceStore(runtime, profileHello); const client = await openClientDbForRun( repoRoot, @@ -1558,21 +1611,29 @@ async function runBenchOnceViaServer( try { await appendInitialOps(client.db, bench.opsA); - const expectedFilterCount = await seedServerState( - runtime, - docId, - bench.opsB, - bench.filter as Filter - ); - if (runtime.waitForOpCount) { - await runtime.waitForOpCount( + if (!preparedFixture) { + const expectedFilterCount = await seedServerState( + runtime, docId, - bench.filter as Filter, - expectedFilterCount, - { timeoutMs: SERVER_READY_TIMEOUT_MS } + bench.opsB, + bench.filter as Filter ); - } else { - await waitForServerOpCount(runtime, docId, bench.opsB.length, directSendThreshold, SERVER_SEED_READY_TIMEOUT_MS); + if (runtime.waitForOpCount) { + await runtime.waitForOpCount( + docId, + bench.filter as Filter, + expectedFilterCount, + { timeoutMs: SERVER_READY_TIMEOUT_MS } + ); + } else { + await waitForServerOpCount( + runtime, + docId, + bench.opsB.length, + directSendThreshold, + SERVER_SEED_READY_TIMEOUT_MS + ); + } } helloTraceStore?.clear(docId); @@ -1663,6 +1724,10 @@ async function runBenchCase( if (includeFirstView && !bench.firstView) { throw new Error(`sync bench workload ${bench.name} does not support --first-view`); } + const preparedFixture = + runtime && canReuseServerFixture(runtime, bench) + ? await prepareServerFixture(runtime, bench, directSendThreshold) + : undefined; for (let i = 0; i < warmupIterations; i += 1) { if (runtime) { @@ -1675,7 +1740,8 @@ async function runBenchCase( profileBackend, profileTransport, profileHello, - directSendThreshold + directSendThreshold, + preparedFixture ); } else { await runBenchOnceDirect( @@ -1704,7 +1770,8 @@ async function runBenchCase( profileBackend, profileTransport, profileHello, - directSendThreshold + directSendThreshold, + preparedFixture ) : await runBenchOnceDirect( repoRoot, @@ -1768,6 +1835,7 @@ async function runBenchCase( codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, + serverFixtureReuse: preparedFixture ? "per-case" : undefined, iterations: iterations > 1 ? iterations : undefined, warmupIterations: warmupIterations > 0 ? warmupIterations : undefined, avgDurationMs: iterations > 1 ? durationMs : undefined, From 22722ff185d76ef3555c7f189932a206caaf8bbc Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sat, 21 Mar 2026 15:30:21 +0100 Subject: [PATCH 06/58] feat(benchmark): cache local sync fixtures across runs --- docs/BENCHMARKS.md | 4 +- .../scripts/bench-sync.ts | 182 ++++++++++++++++-- scripts/run-sync-bench.mjs | 2 + 3 files changed, 172 insertions(+), 16 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index c2c514de..9b317ccf 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -79,7 +79,9 @@ By default, the local sync target runs the Postgres sync server in a spawned chi Local server benchmarks now seed the Postgres backend directly before the timer starts. That keeps the measured path honest, because the actual sync to the client still goes through the real websocket server, while avoiding huge protocol-seed setup costs that are not part of the benchmark question. -For read-only local server workloads, the harness now prepares that server fixture once per benchmark case and reuses it across warmup and measured samples. That removes repeated `50k/100k` Postgres imports from the per-sample path while keeping each client run a fresh-device sync. +For read-only local server workloads, the harness now prepares that server fixture once per benchmark case and reuses it across warmup and measured samples. It also reuses the same seeded Postgres fixture across separate benchmark runs by default when the workload definition matches, so repeated `50k/100k` runs do not keep reimporting the same large server doc. + +Use `--server-fixture-cache=rebuild` when you want to force a fresh local Postgres fixture, or `--server-fixture-cache=off` when you want every run to seed an isolated throwaway fixture. ### Time To First Visible Page diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 35e912b3..17910853 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import net from "node:net"; import path from "node:path"; @@ -21,7 +22,7 @@ import { import { repoRootFromImportMeta, writeResult } from "@treecrdt/benchmark/node"; import type { Operation } from "@treecrdt/interface"; import { nodeIdToBytes16 } from "@treecrdt/interface/ids"; -import { SyncPeer, type Filter } from "@treecrdt/sync"; +import { SyncPeer, type Filter, type SyncBackend } from "@treecrdt/sync"; import { makeQueuedSyncBackend, type FlushableSyncBackend, @@ -52,6 +53,7 @@ type SyncBenchTargetId = | "direct" | "local-postgres-sync-server" | "remote-sync-server"; +type ServerFixtureCacheMode = "off" | "reuse" | "rebuild"; type BenchCase = { storage: StorageKind; @@ -82,6 +84,8 @@ type SyncBenchSample = { type PreparedServerFixture = { docId: string; + cacheKey?: string; + cacheStatus: "disabled" | "hit" | "miss" | "rebuild"; }; type SyncBenchConnection = { @@ -94,6 +98,10 @@ type SyncBenchTargetRuntime = { serverProcess: "child-process" | "in-process" | "remote"; connect: (docId: string) => Promise; seedOps?: (docId: string, ops: Operation[]) => Promise; + inspectDoc?: ( + docId: string + ) => Promise<{ allCount: number; maxLamport: number }>; + resetDoc?: (docId: string) => Promise; waitForOpCount?: ( docId: string, filter: Filter, @@ -203,6 +211,8 @@ const SYNC_BENCH_DIRECT_SEND_THRESHOLD = Math.max( 0, envInt("SYNC_BENCH_DIRECT_SEND_THRESHOLD") ?? 0 ); +const DEFAULT_SERVER_FIXTURE_CACHE_MODE: ServerFixtureCacheMode = "reuse"; +const SYNC_BENCH_SERVER_FIXTURE_CACHE_VERSION = "2026-03-21-v1"; const SERVER_READY_POLL_MS = 100; function envInt(name: string): number | undefined { @@ -435,6 +445,20 @@ function parseDirectSendThreshold(argv: string[]): number { ]) ?? SYNC_BENCH_DIRECT_SEND_THRESHOLD; } +function parseServerFixtureCacheMode(argv: string[]): ServerFixtureCacheMode { + const raw = + parseFlagValue(argv, "--server-fixture-cache") ?? + process.env.SYNC_BENCH_SERVER_FIXTURE_CACHE; + if (!raw) return DEFAULT_SERVER_FIXTURE_CACHE_MODE; + const value = raw.trim().toLowerCase(); + if (value === "off" || value === "reuse" || value === "rebuild") { + return value; + } + throw new Error( + `invalid --server-fixture-cache value "${raw}", expected one of: off, reuse, rebuild` + ); +} + function createEmptyTransportDirectionProfile(): TransportDirectionProfile { return { messages: 0, @@ -957,6 +981,20 @@ async function createLocalPostgresSyncServerTarget( backendModule, postgresUrl ); + const inspectDoc = async (docId: string) => { + const backend = await backendFactory.open(docId); + const [allRefs, currentMaxLamport] = await Promise.all([ + backend.listOpRefs({ all: {} }), + backend.maxLamport(), + ]); + return { + allCount: allRefs.length, + maxLamport: Number(currentMaxLamport), + }; + }; + const resetDoc = async (docId: string) => { + await backendFactory.resetDocForTests(docId); + }; const seedOps = async (docId: string, ops: Operation[]) => { if (ops.length === 0) return; const backend = await backendFactory.open(docId); @@ -981,6 +1019,8 @@ async function createLocalPostgresSyncServerTarget( connect: async (docId) => await connectToSyncServer(`ws://127.0.0.1:${server.port}`, docId), seedOps, + inspectDoc, + resetDoc, waitForOpCount, close: async () => { await server.close(); @@ -1042,6 +1082,8 @@ async function createLocalPostgresSyncServerTarget( connect: async (docId) => await connectToSyncServer(`ws://127.0.0.1:${port}`, docId), seedOps, + inspectDoc, + resetDoc, waitForOpCount, clearHelloTrace: helloTraceStore ? (docId) => helloTraceStore.clear(docId) : undefined, takeHelloTrace: helloTraceStore ? (docId) => helloTraceStore.take(docId) : undefined, @@ -1097,17 +1139,13 @@ async function loadPostgresSyncBackendFactory( backendModule: string, postgresUrl: string ): Promise<{ - open: (docId: string) => Promise<{ - applyOps: (ops: Operation[]) => Promise; - listOpRefs: (filter: Filter) => Promise; - }>; + resetDocForTests: (docId: string) => Promise; + open: (docId: string) => Promise>; }> { const mod = (await import(pathToFileURL(backendModule).href)) as { createPostgresNapiSyncBackendFactory?: (url: string) => { - open: (docId: string) => Promise<{ - applyOps: (ops: Operation[]) => Promise; - listOpRefs: (filter: Filter) => Promise; - }>; + resetDocForTests: (docId: string) => Promise; + open: (docId: string) => Promise>; }; }; if (typeof mod.createPostgresNapiSyncBackendFactory !== "function") { @@ -1369,12 +1407,110 @@ function canReuseServerFixture( ); } +function updateFixtureHashWithBytes( + hash: ReturnType, + value: Uint8Array | null | undefined +): void { + if (value == null) { + hash.update("-1:"); + return; + } + hash.update(`${value.byteLength}:`); + hash.update(Buffer.from(value)); + hash.update(";"); +} + +function updateFixtureHashWithString( + hash: ReturnType, + value: string +): void { + hash.update(`${value.length}:`); + hash.update(value); + hash.update(";"); +} + +function updateFixtureHashWithNumber( + hash: ReturnType, + value: number +): void { + hash.update(`${value};`); +} + +function createServerFixtureCacheKey( + bench: ReturnType +): string { + const hash = createHash("sha256"); + updateFixtureHashWithString(hash, SYNC_BENCH_SERVER_FIXTURE_CACHE_VERSION); + updateFixtureHashWithString(hash, bench.name); + if ("all" in bench.filter) { + updateFixtureHashWithString(hash, "filter:all"); + } else { + updateFixtureHashWithString(hash, "filter:children"); + updateFixtureHashWithBytes(hash, bench.filter.children.parent); + } + for (const op of bench.opsB) { + updateFixtureHashWithBytes(hash, op.meta.id.replica); + updateFixtureHashWithNumber(hash, op.meta.id.counter); + updateFixtureHashWithNumber(hash, op.meta.lamport); + updateFixtureHashWithBytes(hash, op.meta.knownState); + updateFixtureHashWithString(hash, op.kind.type); + switch (op.kind.type) { + case "insert": + updateFixtureHashWithString(hash, op.kind.parent); + updateFixtureHashWithString(hash, op.kind.node); + updateFixtureHashWithBytes(hash, op.kind.orderKey); + updateFixtureHashWithBytes(hash, op.kind.payload); + break; + case "move": + updateFixtureHashWithString(hash, op.kind.node); + updateFixtureHashWithString(hash, op.kind.newParent); + updateFixtureHashWithBytes(hash, op.kind.orderKey); + break; + case "delete": + case "tombstone": + updateFixtureHashWithString(hash, op.kind.node); + break; + case "payload": + updateFixtureHashWithString(hash, op.kind.node); + updateFixtureHashWithBytes(hash, op.kind.payload); + break; + } + } + return hash.digest("hex").slice(0, 24); +} + async function prepareServerFixture( runtime: SyncBenchTargetRuntime, bench: ReturnType, - directSendThreshold: number + directSendThreshold: number, + cacheMode: ServerFixtureCacheMode ): Promise { - const docId = `sqlite-node-sync-bench-${runtime.id}-fixture-${crypto.randomUUID()}`; + const cacheKey = + cacheMode === "off" ? undefined : createServerFixtureCacheKey(bench); + const docId = + cacheMode === "off" + ? `sqlite-node-sync-bench-${runtime.id}-fixture-${crypto.randomUUID()}` + : `sqlite-node-sync-bench-${runtime.id}-fixture-${cacheKey}`; + if (cacheMode === "reuse" && runtime.inspectDoc) { + try { + const current = await runtime.inspectDoc(docId); + if ( + current.allCount === bench.opsB.length && + current.maxLamport === maxLamport(bench.opsB) + ) { + return { + docId, + cacheKey, + cacheStatus: "hit", + }; + } + } catch { + // fall through to rebuild the fixture + } + } + if (cacheMode !== "off") { + await runtime.resetDoc?.(docId); + } const expectedFilterCount = await seedServerState( runtime, docId, @@ -1397,7 +1533,11 @@ async function prepareServerFixture( SERVER_SEED_READY_TIMEOUT_MS ); } - return { docId }; + return { + docId, + cacheKey, + cacheStatus: cacheMode === "rebuild" ? "rebuild" : cacheMode === "reuse" ? "miss" : "disabled", + }; } async function waitForServerOpCount( @@ -1706,7 +1846,8 @@ async function runBenchCase( profileBackend: boolean, profileTransport: boolean, profileHello: boolean, - directSendThreshold: number + directSendThreshold: number, + serverFixtureCacheMode: ServerFixtureCacheMode ): Promise { const bench = buildSyncBenchCase({ workload: benchCase.workload, @@ -1726,7 +1867,7 @@ async function runBenchCase( } const preparedFixture = runtime && canReuseServerFixture(runtime, bench) - ? await prepareServerFixture(runtime, bench, directSendThreshold) + ? await prepareServerFixture(runtime, bench, directSendThreshold, serverFixtureCacheMode) : undefined; for (let i = 0; i < warmupIterations; i += 1) { @@ -1836,6 +1977,15 @@ async function runBenchCase( maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, serverFixtureReuse: preparedFixture ? "per-case" : undefined, + serverFixtureCacheMode: + preparedFixture && serverFixtureCacheMode !== DEFAULT_SERVER_FIXTURE_CACHE_MODE + ? serverFixtureCacheMode + : undefined, + serverFixtureCacheStatus: + preparedFixture && preparedFixture.cacheStatus !== "disabled" + ? preparedFixture.cacheStatus + : undefined, + serverFixtureCacheKey: preparedFixture?.cacheKey, iterations: iterations > 1 ? iterations : undefined, warmupIterations: warmupIterations > 0 ? warmupIterations : undefined, avgDurationMs: iterations > 1 ? durationMs : undefined, @@ -1874,6 +2024,7 @@ async function main() { const profileTransport = parseProfileTransport(argv); const profileHello = parseProfileHello(argv); const directSendThreshold = parseDirectSendThreshold(argv); + const serverFixtureCacheMode = parseServerFixtureCacheMode(argv); const runtimes = await prepareTargetRuntimes( repoRoot, argv, @@ -1914,7 +2065,8 @@ async function main() { profileBackend, profileTransport, profileHello, - directSendThreshold + directSendThreshold, + serverFixtureCacheMode ); const outFile = path.join( repoRoot, diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index 74e3bb57..32dc2efc 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -133,6 +133,7 @@ async function main() { pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 --profile-transport pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --profile-hello pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --direct-send-threshold=64 + pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=50000 --first-view --server-fixture-cache=rebuild TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start Notes: @@ -144,6 +145,7 @@ Notes: - add --iterations=N and --warmup=N to control sample count explicitly; custom --count/--counts runs now default to multiple samples instead of silently dropping to 1 - local sync benches use a spawned child-process server by default for more realistic local vs remote comparisons - local sync benches now use a benchmark-only direct Postgres seed step before timing, so large local runs avoid spending minutes protocol-seeding data that is not part of the measured sync + - local read-only sync benches reuse the same seeded Postgres fixture across warmup, samples, and later matching runs by default; use --server-fixture-cache=rebuild to refresh it or --server-fixture-cache=off to disable that cache - add --profile-backend to capture listOpRefs/getOpsByOpRefs/applyOps timing per backend; on local benches this switches back to the in-process server for debug visibility - add --profile-transport to capture sync message counts, bytes, and a small event timeline - add --profile-hello to capture responder-side hello stage timings; local child-process runs parse server trace output, direct and in-process runs collect it in-process From 74c888f11b4adf4a320502d8e5a4072555cfc0e6 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sat, 21 Mar 2026 15:53:36 +0100 Subject: [PATCH 07/58] feat(benchmark): add local sync fixture priming --- docs/BENCHMARKS.md | 20 ++++ package.json | 1 + .../scripts/bench-sync.ts | 109 ++++++++++++++++++ scripts/run-sync-bench.mjs | 50 +++++++- 4 files changed, 178 insertions(+), 2 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 9b317ccf..a1f9d0aa 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -21,6 +21,7 @@ pnpm benchmark:sqlite-node:note-paths pnpm benchmark:sync pnpm benchmark:sync:direct pnpm benchmark:sync:local +pnpm benchmark:sync:prime pnpm benchmark:sync:remote pnpm benchmark:web pnpm benchmark:wasm @@ -75,6 +76,25 @@ pnpm benchmark:sync:remote -- \ Use `--fanout=20` when you want to model a broader notebook tree. +### Prime Local Fixtures + +Use this when you want to prebuild the local Postgres server fixtures before running the actual local sync benchmarks. + +```sh +pnpm benchmark:sync:prime +``` + +By default this primes the read-only first-view workloads for `10k`, `50k`, and `100k` nodes and forces a rebuild. After that, matching local benchmark runs reuse those fixtures as cache hits instead of reimporting the same large server docs. + +You can still override the forwarded args: + +```sh +pnpm benchmark:sync:prime -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --counts=50000,100000 \ + --server-fixture-cache=rebuild +``` + By default, the local sync target runs the Postgres sync server in a spawned child process so local and remote measurements are closer to each other. When you add `--profile-backend`, the local target intentionally switches to the in-process server so per-backend timings are visible inside the benchmark process. Local server benchmarks now seed the Postgres backend directly before the timer starts. That keeps the measured path honest, because the actual sync to the client still goes through the real websocket server, while avoiding huge protocol-seed setup costs that are not part of the benchmark question. diff --git a/package.json b/package.json index 1bf3b314..127e8c1c 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "benchmark:sync:help": "node scripts/run-sync-bench.mjs --help", "benchmark:sync:direct": "node scripts/run-sync-bench.mjs direct", "benchmark:sync:local": "node scripts/run-sync-bench.mjs local", + "benchmark:sync:prime": "node scripts/run-sync-bench.mjs prime", "benchmark:sync:remote": "node scripts/run-sync-bench.mjs remote", "benchmark:wasm": "pnpm -C packages/treecrdt-wasm-js run benchmark", "benchmark:web": "pnpm -C packages/treecrdt-wa-sqlite/e2e run bench", diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 17910853..001eedd3 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -73,6 +73,11 @@ type SyncBenchResult = { extra?: Record; }; +type PrimedServerFixtureResult = { + name: string; + extra?: Record; +}; + type SyncBenchSample = { totalMs: number; syncMs: number; @@ -459,6 +464,14 @@ function parseServerFixtureCacheMode(argv: string[]): ServerFixtureCacheMode { ); } +function parsePrimeServerFixtures(argv: string[]): boolean { + return parseBooleanFlag( + argv, + "--prime-server-fixtures", + "SYNC_BENCH_PRIME_SERVER_FIXTURES" + ); +} + function createEmptyTransportDirectionProfile(): TransportDirectionProfile { return { messages: 0, @@ -2001,6 +2014,65 @@ async function runBenchCase( }; } +async function primeServerFixtureCase( + benchCase: Omit, + runtimes: Map, SyncBenchTargetRuntime>, + directSendThreshold: number, + serverFixtureCacheMode: ServerFixtureCacheMode +): Promise { + const bench = buildSyncBenchCase({ + workload: benchCase.workload, + size: benchCase.size, + fanout: benchCase.fanout, + }); + const runtime = + benchCase.target === "direct" ? null : runtimes.get(benchCase.target); + if (benchCase.target !== "direct" && !runtime) { + throw new Error(`missing runtime for sync bench target ${benchCase.target}`); + } + if (!runtime) { + throw new Error("server fixture priming requires a sync-server target"); + } + if (!canReuseServerFixture(runtime, bench)) { + throw new Error( + `sync bench workload ${bench.name} does not support reusable local server fixtures` + ); + } + + const preparedFixture = await prepareServerFixture( + runtime, + bench, + directSendThreshold, + serverFixtureCacheMode + ); + return { + name: `${bench.name}-server-fixture`, + extra: { + ...bench.extra, + count: benchCase.size, + fanout: benchCase.fanout, + mode: benchCase.target, + target: benchCase.target, + server: + benchCase.target === "local-postgres-sync-server" + ? "postgres-local" + : benchCase.target === "remote-sync-server" + ? "remote" + : "none", + serverProcess: runtime.serverProcess, + measurement: "server-fixture-prime", + directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, + serverFixtureReuse: "per-case", + serverFixtureCacheMode, + serverFixtureCacheStatus: preparedFixture.cacheStatus, + serverFixtureCacheKey: preparedFixture.cacheKey, + docId: preparedFixture.docId, + fixtureOpCount: bench.opsB.length, + fixtureMaxLamport: maxLamport(bench.opsB), + }, + }; +} + async function main() { const argv = process.argv.slice(2); const repoRoot = repoRootFromImportMeta(import.meta.url, 3); @@ -2025,6 +2097,7 @@ async function main() { const profileHello = parseProfileHello(argv); const directSendThreshold = parseDirectSendThreshold(argv); const serverFixtureCacheMode = parseServerFixtureCacheMode(argv); + const primeServerFixtures = parsePrimeServerFixtures(argv); const runtimes = await prepareTargetRuntimes( repoRoot, argv, @@ -2035,6 +2108,42 @@ async function main() { ); try { + if (primeServerFixtures) { + if (targets.some((target) => target !== "local-postgres-sync-server")) { + throw new Error( + "--prime-server-fixtures only supports the local-postgres-sync-server target" + ); + } + const primeCases = workloads.flatMap((workload) => { + const entries = + workload === "sync-root-children-fanout10" ? rootConfig : config; + return entries.map(([size]) => ({ + target: "local-postgres-sync-server" as const, + workload, + size, + fanout, + })); + }); + + for (const primeCase of primeCases) { + const result = await primeServerFixtureCase( + primeCase, + runtimes, + directSendThreshold, + serverFixtureCacheMode + ); + console.log( + JSON.stringify({ + implementation: "sqlite-node", + workload: result.name, + target: primeCase.target, + extra: result.extra, + }) + ); + } + return; + } + const cases: BenchCase[] = []; for (const target of targets) { for (const storage of storages) { diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index 32dc2efc..b075ed01 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -60,6 +60,14 @@ function parseRequestedTargets(args) { return ["direct"]; } +function hasExplicitTarget(args) { + if (extractFlagValue(args, "--targets") ?? extractFlagValue(args, "--target")) { + return true; + } + const positional = args.find((arg) => !arg.startsWith("--")); + return positional ? normalizeTarget(positional) != null : false; +} + function stripPositionalTargetArg(args) { let removed = false; return args.filter((arg) => { @@ -71,10 +79,24 @@ function stripPositionalTargetArg(args) { }); } +function stripPrimeArg(args) { + let removed = false; + return args.filter((arg) => { + if (removed) return true; + if (arg !== "prime") return true; + removed = true; + return false; + }); +} + function hasFlag(args, flag) { return args.some((arg) => arg.startsWith(`${flag}=`)); } +function hasAnyFlag(args, flags) { + return flags.some((flag) => hasFlag(args, flag)); +} + async function runPnpm(args, env) { await new Promise((resolve, reject) => { const child = spawn("pnpm", args, { @@ -122,9 +144,12 @@ function effectiveEnv(targets, args) { async function main() { const rawArgs = process.argv.slice(2); + const primeMode = + rawArgs.includes("prime") || rawArgs.includes("--prime-server-fixtures"); if (rawArgs.includes("--help")) { console.log(`Usage: pnpm benchmark:sync + pnpm benchmark:sync:prime pnpm benchmark:sync:direct -- --workloads=sync-balanced-children-payloads-cold-start --counts=10000,50000 --fanout=10 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view @@ -134,6 +159,7 @@ async function main() { pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --profile-hello pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --direct-send-threshold=64 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=50000 --first-view --server-fixture-cache=rebuild + pnpm benchmark:sync:prime -- --counts=10000,50000,100000 TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start Notes: @@ -146,6 +172,7 @@ Notes: - local sync benches use a spawned child-process server by default for more realistic local vs remote comparisons - local sync benches now use a benchmark-only direct Postgres seed step before timing, so large local runs avoid spending minutes protocol-seeding data that is not part of the measured sync - local read-only sync benches reuse the same seeded Postgres fixture across warmup, samples, and later matching runs by default; use --server-fixture-cache=rebuild to refresh it or --server-fixture-cache=off to disable that cache + - pnpm benchmark:sync:prime warms local Postgres fixtures for the read-only first-view workloads and defaults to rebuilding them for 10k/50k/100k unless you override the forwarded args - add --profile-backend to capture listOpRefs/getOpsByOpRefs/applyOps timing per backend; on local benches this switches back to the in-process server for debug visibility - add --profile-transport to capture sync message counts, bytes, and a small event timeline - add --profile-hello to capture responder-side hello stage timings; local child-process runs parse server trace output, direct and in-process runs collect it in-process @@ -154,8 +181,27 @@ Notes: return; } - const targets = parseRequestedTargets(rawArgs); - const forwardedArgs = stripPositionalTargetArg(rawArgs); + const baseArgs = stripPrimeArg(rawArgs); + const explicitTarget = hasExplicitTarget(baseArgs); + const targets = + primeMode && !explicitTarget + ? ["local-postgres-sync-server"] + : parseRequestedTargets(baseArgs); + const forwardedArgs = stripPositionalTargetArg(baseArgs); + if (primeMode && !forwardedArgs.includes("--prime-server-fixtures")) { + forwardedArgs.push("--prime-server-fixtures"); + } + if (primeMode && !hasAnyFlag(forwardedArgs, ["--workloads", "--workload"])) { + forwardedArgs.push( + "--workloads=sync-balanced-children-cold-start,sync-balanced-children-payloads-cold-start" + ); + } + if (primeMode && !hasAnyFlag(forwardedArgs, ["--counts", "--count"])) { + forwardedArgs.push("--counts=10000,50000,100000"); + } + if (primeMode && !hasFlag(forwardedArgs, "--server-fixture-cache")) { + forwardedArgs.push("--server-fixture-cache=rebuild"); + } const env = effectiveEnv(targets, forwardedArgs); const buildCommands = [ From 550378917b796112c27db53e7a89f1b2cd886889 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sat, 21 Mar 2026 22:32:07 +0100 Subject: [PATCH 08/58] fix(benchmark): use builtin websocket for remote sync benches --- .../scripts/bench-sync.ts | 123 +++++++++++++++++- 1 file changed, 118 insertions(+), 5 deletions(-) diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 001eedd3..2ab1370d 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -1,5 +1,6 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; +import { setMaxListeners } from "node:events"; import fs from "node:fs/promises"; import net from "node:net"; import path from "node:path"; @@ -802,6 +803,32 @@ async function closeWebSocket(ws: WebSocket): Promise { }); } +type BuiltinWebSocket = InstanceType; + +async function closeBuiltinWebSocket(ws: BuiltinWebSocket): Promise { + if (ws.readyState === globalThis.WebSocket.CLOSED) return; + await new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + ws.removeEventListener("close", onClose); + ws.removeEventListener("error", onError); + resolve(); + }; + const onClose = () => finish(); + const onError = () => finish(); + ws.addEventListener("close", onClose); + ws.addEventListener("error", onError); + try { + ws.close(); + } catch { + finish(); + } + setTimeout(finish, 1_000); + }); +} + async function openWebSocket(url: URL): Promise { return await new Promise((resolve, reject) => { const ws = new WebSocket(url); @@ -835,6 +862,47 @@ async function openWebSocket(url: URL): Promise { }); } +async function openBuiltinWebSocket(url: URL): Promise { + return await new Promise((resolve, reject) => { + const WebSocketCtor = globalThis.WebSocket; + if (typeof WebSocketCtor !== "function") { + reject(new Error("global WebSocket is not available in this Node runtime")); + return; + } + + const ws = new WebSocketCtor(url.toString()); + setMaxListeners(0, ws); + ws.binaryType = "arraybuffer"; + let settled = false; + + const finish = ( + fn: (value: BuiltinWebSocket | Error) => void, + value: BuiltinWebSocket | Error + ) => { + if (settled) return; + settled = true; + clearTimeout(timer); + ws.removeEventListener("open", onOpen); + ws.removeEventListener("error", onError); + fn(value); + }; + + const onOpen = () => finish(resolve, ws); + const onError = () => { + void closeBuiltinWebSocket(ws); + finish(reject, new Error(`failed connecting to ${url.toString()}`)); + }; + + const timer = setTimeout(() => { + void closeBuiltinWebSocket(ws); + finish(reject, new Error(`timed out connecting to ${url.toString()}`)); + }, 5_000); + + ws.addEventListener("open", onOpen); + ws.addEventListener("error", onError); + }); +} + function createNodeWebSocketTransport( ws: WebSocket ): DuplexTransport { @@ -867,6 +935,38 @@ function createNodeWebSocketTransport( }; } +function createBuiltinWebSocketTransport( + ws: BuiltinWebSocket +): DuplexTransport { + return { + send: async (bytes) => { + try { + ws.send(bytes); + } catch (error) { + throw error instanceof Error ? error : new Error(String(error)); + } + }, + onMessage: (handler) => { + const onMessage = (event: { data: unknown }) => { + const { data } = event; + if (data instanceof Uint8Array) { + handler(data); + } else if (data instanceof ArrayBuffer) { + handler(new Uint8Array(data)); + } else if (ArrayBuffer.isView(data)) { + handler(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)); + } else if (typeof data === "string") { + handler(Buffer.from(data)); + } else { + throw new Error(`unsupported builtin WebSocket message type: ${typeof data}`); + } + }; + ws.addEventListener("message", onMessage as EventListener); + return () => ws.removeEventListener("message", onMessage as EventListener); + }, + }; +} + async function makeBackend(opts: { db: Database.Database; docId: string; @@ -945,11 +1045,19 @@ async function measureFirstViewAfterSync( async function connectToSyncServer( baseUrl: string, - docId: string + docId: string, + opts?: { client?: "ws-package" | "builtin" } ): Promise { const url = normalizeSyncServerUrl(baseUrl, docId); - const ws = await openWebSocket(url); - const wire = createNodeWebSocketTransport(ws); + const client = opts?.client ?? "ws-package"; + const ws = + client === "builtin" + ? await openBuiltinWebSocket(url) + : await openWebSocket(url); + const wire = + client === "builtin" + ? createBuiltinWebSocketTransport(ws) + : createNodeWebSocketTransport(ws); const transport = wrapDuplexTransportWithCodec( wire, treecrdtSyncV0ProtobufCodec as any @@ -957,7 +1065,11 @@ async function connectToSyncServer( return { transport, close: async () => { - await closeWebSocket(ws); + if (client === "builtin") { + await closeBuiltinWebSocket(ws); + } else { + await closeWebSocket(ws); + } }, }; } @@ -1196,7 +1308,8 @@ function createRemoteSyncServerTarget( return { id: "remote-sync-server", serverProcess: "remote", - connect: async (docId) => await connectToSyncServer(baseUrl, docId), + connect: async (docId) => + await connectToSyncServer(baseUrl, docId, { client: "builtin" }), close: async () => {}, }; } From c080c9665d55b72ad68d03f8541d53012c40545d Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 00:46:16 +0100 Subject: [PATCH 09/58] fix(riblt-wasm): disable wasm-opt in release builds --- packages/treecrdt-riblt-wasm/Cargo.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/treecrdt-riblt-wasm/Cargo.toml b/packages/treecrdt-riblt-wasm/Cargo.toml index f85994c0..91e5c3c3 100644 --- a/packages/treecrdt-riblt-wasm/Cargo.toml +++ b/packages/treecrdt-riblt-wasm/Cargo.toml @@ -5,6 +5,12 @@ edition = "2021" license = "MIT" description = "WASM bridge for Rateless IBLT (riblt) used by TreeCRDT sync." +[package.metadata.wasm-pack.profile.release] +# Avoid wasm-opt on release builds. The optimized bundle has been observed to +# crash at startup in the deployed Node/Linux ARM environment while +# initializing the wasm-bindgen externref table. +wasm-opt = false + [lib] crate-type = ["cdylib", "rlib"] @@ -13,4 +19,3 @@ riblt = { git = "https://github.com/Intersubjective/riblt-rust", rev = "23cd9ce6 serde = { version = "1", features = ["derive"] } serde-wasm-bindgen = "0.6" wasm-bindgen = { version = "0.2", features = ["serde-serialize"] } - From 01ce273e8b4313fa26f887df29b9b0c1855725b1 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 01:15:05 +0100 Subject: [PATCH 10/58] fix(benchmark): avoid recursive remote websocket cleanup --- packages/treecrdt-sqlite-node/scripts/bench-sync.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 2ab1370d..ca124b75 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -821,7 +821,9 @@ async function closeBuiltinWebSocket(ws: BuiltinWebSocket): Promise { ws.addEventListener("close", onClose); ws.addEventListener("error", onError); try { - ws.close(); + if (ws.readyState !== globalThis.WebSocket.CLOSING) { + ws.close(); + } } catch { finish(); } @@ -889,12 +891,17 @@ async function openBuiltinWebSocket(url: URL): Promise { const onOpen = () => finish(resolve, ws); const onError = () => { - void closeBuiltinWebSocket(ws); finish(reject, new Error(`failed connecting to ${url.toString()}`)); }; const timer = setTimeout(() => { - void closeBuiltinWebSocket(ws); + try { + if (ws.readyState !== globalThis.WebSocket.CLOSING) { + ws.close(); + } + } catch { + // Ignore close failures during connection timeout cleanup. + } finish(reject, new Error(`timed out connecting to ${url.toString()}`)); }, 5_000); From f6c9e1316cdbb0a8ec68c96ba76628b0b06af745 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 08:59:11 +0100 Subject: [PATCH 11/58] fix(sync-server): use dedicated readiness pg client --- packages/sync/server/postgres-node/src/server.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/sync/server/postgres-node/src/server.ts b/packages/sync/server/postgres-node/src/server.ts index a327c7a7..53612014 100644 --- a/packages/sync/server/postgres-node/src/server.ts +++ b/packages/sync/server/postgres-node/src/server.ts @@ -564,18 +564,7 @@ export function createPostgresNodeDocStore(opts: PostgresNodeDocStoreOptions): P }; } -async function createReadinessProbe( - postgresUrl: string, - docUpdateBus: PostgresDocUpdateBus | undefined -): Promise { - if (docUpdateBus) { - return { - check: async () => { - await withTimeout(docUpdateBus.ping(), 3_000, "postgres readiness ping"); - }, - }; - } - +async function createReadinessProbe(postgresUrl: string): Promise { const client = new PgClient({ connectionString: postgresUrl }); await client.connect(); return { @@ -683,7 +672,7 @@ export async function startSyncServer(opts: SyncServerOptions): Promise Date: Sun, 22 Mar 2026 09:28:22 +0100 Subject: [PATCH 12/58] feat(benchmark): tune sync ops batch size --- docs/BENCHMARKS.md | 12 +++ .../scripts/bench-sync.ts | 79 ++++++++++++++----- scripts/run-sync-bench.mjs | 2 + 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index a1f9d0aa..d71a4b09 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -139,6 +139,18 @@ pnpm benchmark:sync:local -- \ --direct-send-threshold=64 ``` +Add `--max-ops-per-batch=N` when you want to force smaller `opsBatch` messages. This is useful for stress-testing large upload paths and for debugging remote seed behavior where very large inbound batches may monopolize a server task. + +```sh +TREECRDT_SYNC_SERVER_URL=ws://host/sync \ +pnpm benchmark:sync:remote -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --first-view \ + --direct-send-threshold=64 \ + --max-ops-per-batch=500 +``` + ### Backend Call Profiling Add `--profile-backend` when you want per-backend timings for: diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index ca124b75..9490ec42 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -217,6 +217,7 @@ const SYNC_BENCH_DIRECT_SEND_THRESHOLD = Math.max( 0, envInt("SYNC_BENCH_DIRECT_SEND_THRESHOLD") ?? 0 ); +const SYNC_BENCH_MAX_OPS_PER_BATCH = envInt("SYNC_BENCH_MAX_OPS_PER_BATCH"); const DEFAULT_SERVER_FIXTURE_CACHE_MODE: ServerFixtureCacheMode = "reuse"; const SYNC_BENCH_SERVER_FIXTURE_CACHE_VERSION = "2026-03-21-v1"; const SERVER_READY_POLL_MS = 100; @@ -451,6 +452,12 @@ function parseDirectSendThreshold(argv: string[]): number { ]) ?? SYNC_BENCH_DIRECT_SEND_THRESHOLD; } +function parseMaxOpsPerBatch(argv: string[]): number | undefined { + return parseOptionalPositiveIntFlag(argv, "--max-ops-per-batch", [ + "SYNC_BENCH_MAX_OPS_PER_BATCH", + ]) ?? SYNC_BENCH_MAX_OPS_PER_BATCH; +} + function parseServerFixtureCacheMode(argv: string[]): ServerFixtureCacheMode { const raw = parseFlagValue(argv, "--server-fixture-cache") ?? @@ -1451,11 +1458,13 @@ async function syncBackendThroughServer( maxCodewords?: number; codewordsPerMessage?: number; directSendThreshold?: number; + maxOpsPerBatch?: number; } = {} ): Promise { const peer = new SyncPeer(backend, { maxCodewords: opts.maxCodewords ?? SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold: opts.directSendThreshold ?? 0, + ...(opts.maxOpsPerBatch != null ? { maxOpsPerBatch: opts.maxOpsPerBatch } : {}), }); const connection = await runtime.connect(docId); const detach = peer.attach(connection.transport); @@ -1464,6 +1473,7 @@ async function syncBackendThroughServer( await peer.syncOnce(connection.transport, filter, { maxCodewords: opts.maxCodewords ?? SYNC_BENCH_DEFAULT_MAX_CODEWORDS, codewordsPerMessage: opts.codewordsPerMessage ?? SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + ...(opts.maxOpsPerBatch != null ? { maxOpsPerBatch: opts.maxOpsPerBatch } : {}), }); await backend.flush(); } finally { @@ -1476,7 +1486,8 @@ async function seedServerState( runtime: SyncBenchTargetRuntime, docId: string, ops: Operation[], - filter: Filter + filter: Filter, + maxOpsPerBatch?: number ): Promise { if (ops.length === 0) return 0; @@ -1501,6 +1512,7 @@ async function seedServerState( const peer = new SyncPeer(seedBackend, { maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, directSendThreshold: 0, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); const connection = await runtime.connect(docId); const detach = peer.attach(connection.transport); @@ -1508,6 +1520,7 @@ async function seedServerState( await peer.syncOnce(connection.transport, { all: {} }, { maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); await seedBackend.flush(); if (runtime.waitForOpCount) { @@ -1616,7 +1629,8 @@ async function prepareServerFixture( runtime: SyncBenchTargetRuntime, bench: ReturnType, directSendThreshold: number, - cacheMode: ServerFixtureCacheMode + cacheMode: ServerFixtureCacheMode, + maxOpsPerBatch?: number ): Promise { const cacheKey = cacheMode === "off" ? undefined : createServerFixtureCacheKey(bench); @@ -1648,7 +1662,8 @@ async function prepareServerFixture( runtime, docId, bench.opsB, - bench.filter as Filter + bench.filter as Filter, + maxOpsPerBatch ); if (runtime.waitForOpCount) { await runtime.waitForOpCount( @@ -1658,13 +1673,14 @@ async function prepareServerFixture( { timeoutMs: SERVER_READY_TIMEOUT_MS } ); } else { - await waitForServerOpCount( - runtime, - docId, - bench.opsB.length, - directSendThreshold, - SERVER_SEED_READY_TIMEOUT_MS - ); + await waitForServerOpCount( + runtime, + docId, + bench.opsB.length, + directSendThreshold, + maxOpsPerBatch, + SERVER_SEED_READY_TIMEOUT_MS + ); } return { docId, @@ -1678,6 +1694,7 @@ async function waitForServerOpCount( docId: string, expectedCount: number, directSendThreshold: number, + maxOpsPerBatch?: number, timeoutMs = SERVER_READY_TIMEOUT_MS ): Promise { const deadline = Date.now() + timeoutMs; @@ -1693,6 +1710,7 @@ async function waitForServerOpCount( await syncBackendThroughServer(runtime, docId, verifierBackend, { all: {} }, { maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, directSendThreshold, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); lastCount = countOps(verifierDb); if (lastCount === expectedCount) { @@ -1749,7 +1767,8 @@ async function runBenchOnceDirect( profileBackend: boolean, profileTransport: boolean, profileHello: boolean, - directSendThreshold: number + directSendThreshold: number, + maxOpsPerBatch?: number ): Promise { const runId = crypto.randomUUID(); const docId = `sqlite-node-sync-bench-${runId}`; @@ -1800,10 +1819,12 @@ async function runBenchOnceDirect( const pa = new SyncPeer(backendA, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); const pb = new SyncPeer(backendB, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); const detachA = pa.attach(transportA); const detachB = pb.attach(transportB); @@ -1815,6 +1836,7 @@ async function runBenchOnceDirect( await pa.syncOnce(transportA, bench.filter as Filter, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); await Promise.all([backendA.flush(), backendB.flush()]); const syncedAt = performance.now(); @@ -1866,6 +1888,7 @@ async function runBenchOnceViaServer( profileTransport: boolean, profileHello: boolean, directSendThreshold: number, + maxOpsPerBatch: number | undefined, preparedFixture?: PreparedServerFixture ): Promise { const runId = crypto.randomUUID(); @@ -1889,7 +1912,8 @@ async function runBenchOnceViaServer( runtime, docId, bench.opsB, - bench.filter as Filter + bench.filter as Filter, + maxOpsPerBatch ); if (runtime.waitForOpCount) { await runtime.waitForOpCount( @@ -1904,6 +1928,7 @@ async function runBenchOnceViaServer( docId, bench.opsB.length, directSendThreshold, + maxOpsPerBatch, SERVER_SEED_READY_TIMEOUT_MS ); } @@ -1920,6 +1945,7 @@ async function runBenchOnceViaServer( const peer = new SyncPeer(clientBackend, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); const connection = await runtime.connect(docId); const transportProfile = profileTransport @@ -1934,6 +1960,7 @@ async function runBenchOnceViaServer( await peer.syncOnce(transport, bench.filter as Filter, { maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); await clientBackend.flush(); const syncedAt = performance.now(); @@ -1980,7 +2007,8 @@ async function runBenchCase( profileTransport: boolean, profileHello: boolean, directSendThreshold: number, - serverFixtureCacheMode: ServerFixtureCacheMode + serverFixtureCacheMode: ServerFixtureCacheMode, + maxOpsPerBatch?: number ): Promise { const bench = buildSyncBenchCase({ workload: benchCase.workload, @@ -2000,7 +2028,7 @@ async function runBenchCase( } const preparedFixture = runtime && canReuseServerFixture(runtime, bench) - ? await prepareServerFixture(runtime, bench, directSendThreshold, serverFixtureCacheMode) + ? await prepareServerFixture(runtime, bench, directSendThreshold, serverFixtureCacheMode, maxOpsPerBatch) : undefined; for (let i = 0; i < warmupIterations; i += 1) { @@ -2015,6 +2043,7 @@ async function runBenchCase( profileTransport, profileHello, directSendThreshold, + maxOpsPerBatch, preparedFixture ); } else { @@ -2026,7 +2055,8 @@ async function runBenchCase( profileBackend, profileTransport, profileHello, - directSendThreshold + directSendThreshold, + maxOpsPerBatch ); } } @@ -2045,6 +2075,7 @@ async function runBenchCase( profileTransport, profileHello, directSendThreshold, + maxOpsPerBatch, preparedFixture ) : await runBenchOnceDirect( @@ -2055,7 +2086,8 @@ async function runBenchCase( profileBackend, profileTransport, profileHello, - directSendThreshold + directSendThreshold, + maxOpsPerBatch ) ); } @@ -2109,6 +2141,7 @@ async function runBenchCase( codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, + maxOpsPerBatch, serverFixtureReuse: preparedFixture ? "per-case" : undefined, serverFixtureCacheMode: preparedFixture && serverFixtureCacheMode !== DEFAULT_SERVER_FIXTURE_CACHE_MODE @@ -2138,7 +2171,8 @@ async function primeServerFixtureCase( benchCase: Omit, runtimes: Map, SyncBenchTargetRuntime>, directSendThreshold: number, - serverFixtureCacheMode: ServerFixtureCacheMode + serverFixtureCacheMode: ServerFixtureCacheMode, + maxOpsPerBatch?: number ): Promise { const bench = buildSyncBenchCase({ workload: benchCase.workload, @@ -2163,7 +2197,8 @@ async function primeServerFixtureCase( runtime, bench, directSendThreshold, - serverFixtureCacheMode + serverFixtureCacheMode, + maxOpsPerBatch ); return { name: `${bench.name}-server-fixture`, @@ -2182,6 +2217,7 @@ async function primeServerFixtureCase( serverProcess: runtime.serverProcess, measurement: "server-fixture-prime", directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, + maxOpsPerBatch, serverFixtureReuse: "per-case", serverFixtureCacheMode, serverFixtureCacheStatus: preparedFixture.cacheStatus, @@ -2216,6 +2252,7 @@ async function main() { const profileTransport = parseProfileTransport(argv); const profileHello = parseProfileHello(argv); const directSendThreshold = parseDirectSendThreshold(argv); + const maxOpsPerBatch = parseMaxOpsPerBatch(argv); const serverFixtureCacheMode = parseServerFixtureCacheMode(argv); const primeServerFixtures = parsePrimeServerFixtures(argv); const runtimes = await prepareTargetRuntimes( @@ -2250,7 +2287,8 @@ async function main() { primeCase, runtimes, directSendThreshold, - serverFixtureCacheMode + serverFixtureCacheMode, + maxOpsPerBatch ); console.log( JSON.stringify({ @@ -2295,7 +2333,8 @@ async function main() { profileTransport, profileHello, directSendThreshold, - serverFixtureCacheMode + serverFixtureCacheMode, + maxOpsPerBatch ); const outFile = path.join( repoRoot, diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index b075ed01..17068f87 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -158,6 +158,7 @@ async function main() { pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 --profile-transport pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --profile-hello pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --direct-send-threshold=64 + pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --direct-send-threshold=64 --max-ops-per-batch=500 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=50000 --first-view --server-fixture-cache=rebuild pnpm benchmark:sync:prime -- --counts=10000,50000,100000 TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start @@ -177,6 +178,7 @@ Notes: - add --profile-transport to capture sync message counts, bytes, and a small event timeline - add --profile-hello to capture responder-side hello stage timings; local child-process runs parse server trace output, direct and in-process runs collect it in-process - add --direct-send-threshold=N to experiment with a clean-slate shortcut that skips the RIBLT round when the requested local filter is empty and the responder has at most N scoped ops + - add --max-ops-per-batch=N to force smaller opsBatch messages when stress-testing upload paths or remote seed behavior - extra args are forwarded to packages/treecrdt-sqlite-node/scripts/bench-sync.ts`); return; } From e78bad75a2135048069c5f73b67e9b61c96a5f8a Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 10:18:25 +0100 Subject: [PATCH 13/58] feat(benchmark): add post-seed cooldown probe --- .../scripts/bench-sync.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 9490ec42..004af54c 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -218,6 +218,10 @@ const SYNC_BENCH_DIRECT_SEND_THRESHOLD = Math.max( envInt("SYNC_BENCH_DIRECT_SEND_THRESHOLD") ?? 0 ); const SYNC_BENCH_MAX_OPS_PER_BATCH = envInt("SYNC_BENCH_MAX_OPS_PER_BATCH"); +const SYNC_BENCH_POST_SEED_WAIT_MS = Math.max( + 0, + envInt("SYNC_BENCH_POST_SEED_WAIT_MS") ?? 0 +); const DEFAULT_SERVER_FIXTURE_CACHE_MODE: ServerFixtureCacheMode = "reuse"; const SYNC_BENCH_SERVER_FIXTURE_CACHE_VERSION = "2026-03-21-v1"; const SERVER_READY_POLL_MS = 100; @@ -458,6 +462,14 @@ function parseMaxOpsPerBatch(argv: string[]): number | undefined { ]) ?? SYNC_BENCH_MAX_OPS_PER_BATCH; } +function parsePostSeedWaitMs(argv: string[]): number { + return ( + parseOptionalNonNegativeIntFlag(argv, "--post-seed-wait-ms", [ + "SYNC_BENCH_POST_SEED_WAIT_MS", + ]) ?? SYNC_BENCH_POST_SEED_WAIT_MS + ); +} + function parseServerFixtureCacheMode(argv: string[]): ServerFixtureCacheMode { const raw = parseFlagValue(argv, "--server-fixture-cache") ?? @@ -1889,6 +1901,7 @@ async function runBenchOnceViaServer( profileHello: boolean, directSendThreshold: number, maxOpsPerBatch: number | undefined, + postSeedWaitMs: number, preparedFixture?: PreparedServerFixture ): Promise { const runId = crypto.randomUUID(); @@ -1932,6 +1945,9 @@ async function runBenchOnceViaServer( SERVER_SEED_READY_TIMEOUT_MS ); } + if (postSeedWaitMs > 0) { + await sleep(postSeedWaitMs); + } } helloTraceStore?.clear(docId); @@ -2008,6 +2024,7 @@ async function runBenchCase( profileHello: boolean, directSendThreshold: number, serverFixtureCacheMode: ServerFixtureCacheMode, + postSeedWaitMs: number, maxOpsPerBatch?: number ): Promise { const bench = buildSyncBenchCase({ @@ -2044,6 +2061,7 @@ async function runBenchCase( profileHello, directSendThreshold, maxOpsPerBatch, + postSeedWaitMs, preparedFixture ); } else { @@ -2076,6 +2094,7 @@ async function runBenchCase( profileHello, directSendThreshold, maxOpsPerBatch, + postSeedWaitMs, preparedFixture ) : await runBenchOnceDirect( @@ -2142,6 +2161,7 @@ async function runBenchCase( maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS, directSendThreshold: directSendThreshold > 0 ? directSendThreshold : undefined, maxOpsPerBatch, + postSeedWaitMs: postSeedWaitMs > 0 ? postSeedWaitMs : undefined, serverFixtureReuse: preparedFixture ? "per-case" : undefined, serverFixtureCacheMode: preparedFixture && serverFixtureCacheMode !== DEFAULT_SERVER_FIXTURE_CACHE_MODE @@ -2253,6 +2273,7 @@ async function main() { const profileHello = parseProfileHello(argv); const directSendThreshold = parseDirectSendThreshold(argv); const maxOpsPerBatch = parseMaxOpsPerBatch(argv); + const postSeedWaitMs = parsePostSeedWaitMs(argv); const serverFixtureCacheMode = parseServerFixtureCacheMode(argv); const primeServerFixtures = parsePrimeServerFixtures(argv); const runtimes = await prepareTargetRuntimes( @@ -2334,6 +2355,7 @@ async function main() { profileHello, directSendThreshold, serverFixtureCacheMode, + postSeedWaitMs, maxOpsPerBatch ); const outFile = path.join( From f77f2680944dc4bdfed9fa6ad57a873f1b28767d Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 10:38:49 +0100 Subject: [PATCH 14/58] fix(benchmark): avoid full remote verifier sync --- packages/treecrdt-sqlite-node/scripts/bench-sync.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 004af54c..7ef0e4ee 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -1688,7 +1688,8 @@ async function prepareServerFixture( await waitForServerOpCount( runtime, docId, - bench.opsB.length, + bench.filter as Filter, + expectedFilterCount, directSendThreshold, maxOpsPerBatch, SERVER_SEED_READY_TIMEOUT_MS @@ -1704,6 +1705,7 @@ async function prepareServerFixture( async function waitForServerOpCount( runtime: SyncBenchTargetRuntime, docId: string, + filter: Filter, expectedCount: number, directSendThreshold: number, maxOpsPerBatch?: number, @@ -1719,7 +1721,7 @@ async function waitForServerOpCount( docId, initialMaxLamport: 0, }); - await syncBackendThroughServer(runtime, docId, verifierBackend, { all: {} }, { + await syncBackendThroughServer(runtime, docId, verifierBackend, filter, { maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, directSendThreshold, ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), @@ -1939,7 +1941,8 @@ async function runBenchOnceViaServer( await waitForServerOpCount( runtime, docId, - bench.opsB.length, + bench.filter as Filter, + expectedFilterCount, directSendThreshold, maxOpsPerBatch, SERVER_SEED_READY_TIMEOUT_MS From db41f4013ed0793844d4235232eae63eb2f6089b Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 10:38:49 +0100 Subject: [PATCH 15/58] fix(sync-server): avoid overlapping pg control queries --- .../sync/server/postgres-node/src/server.ts | 18 ++--- .../postgres-node/tests/pg-notify-bus.test.ts | 70 +++++++++++++++++++ 2 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 packages/sync/server/postgres-node/tests/pg-notify-bus.test.ts diff --git a/packages/sync/server/postgres-node/src/server.ts b/packages/sync/server/postgres-node/src/server.ts index 53612014..df904216 100644 --- a/packages/sync/server/postgres-node/src/server.ts +++ b/packages/sync/server/postgres-node/src/server.ts @@ -23,7 +23,7 @@ import type { WebSocketSyncServerUpgradeHook, } from "@treecrdt/sync-server-core"; import { startWebSocketSyncServer } from "@treecrdt/sync-server-core"; -import { Client as PgClient } from "pg"; +import { Client as PgClient, Pool as PgPool } from "pg"; type Awaitable = T | Promise; @@ -164,16 +164,16 @@ type DocUpdatePayload = { source?: string; }; -class PostgresDocUpdateBus { +export class PostgresDocUpdateBus { private readonly channel: string; - private readonly queryClient: PgClient; + private readonly queryPool: PgPool; private readonly listenClient: PgClient; private readonly sourceId: string; private closed = false; private constructor(private readonly opts: PostgresDocUpdateBusOptions) { this.channel = ensurePostgresChannelName(opts.channel); - this.queryClient = new PgClient({ connectionString: opts.postgresUrl }); + this.queryPool = new PgPool({ connectionString: opts.postgresUrl }); this.listenClient = new PgClient({ connectionString: opts.postgresUrl }); this.sourceId = randomUUID(); } @@ -185,7 +185,7 @@ class PostgresDocUpdateBus { } private async start(): Promise { - await Promise.all([this.queryClient.connect(), this.listenClient.connect()]); + await this.listenClient.connect(); this.listenClient.on("notification", (msg: { channel: string; payload?: string }) => { if (msg.channel !== this.channel) return; @@ -205,18 +205,18 @@ class PostgresDocUpdateBus { } async hasDoc(docId: string): Promise { - const res = await this.queryClient.query("SELECT 1 FROM treecrdt_meta WHERE doc_id = $1 LIMIT 1", [docId]); + const res = await this.queryPool.query("SELECT 1 FROM treecrdt_meta WHERE doc_id = $1 LIMIT 1", [docId]); return (res.rowCount ?? 0) > 0; } async publishDocUpdate(docId: string): Promise { if (this.closed || docId.length === 0) return; const payload = JSON.stringify({ docId, source: this.sourceId } satisfies DocUpdatePayload); - await this.queryClient.query("SELECT pg_notify($1, $2)", [this.channel, payload]); + await this.queryPool.query("SELECT pg_notify($1, $2)", [this.channel, payload]); } async ping(): Promise { - await this.queryClient.query("SELECT 1"); + await this.queryPool.query("SELECT 1"); } async close(): Promise { @@ -227,7 +227,7 @@ class PostgresDocUpdateBus { } catch { // ignore } - await Promise.allSettled([this.listenClient.end(), this.queryClient.end()]); + await Promise.allSettled([this.listenClient.end(), this.queryPool.end()]); } } diff --git a/packages/sync/server/postgres-node/tests/pg-notify-bus.test.ts b/packages/sync/server/postgres-node/tests/pg-notify-bus.test.ts new file mode 100644 index 00000000..41e9dd69 --- /dev/null +++ b/packages/sync/server/postgres-node/tests/pg-notify-bus.test.ts @@ -0,0 +1,70 @@ +import { beforeEach, expect, test, vi } from "vitest"; + +const pgMockState = { + clientActiveQueries: 0, + clientQueries: [] as string[], + poolQueries: [] as string[], +}; + +vi.mock("pg", () => { + class MockPool { + async query(sql: string): Promise<{ rowCount: number }> { + pgMockState.poolQueries.push(sql); + await new Promise((resolve) => setTimeout(resolve, 5)); + return { rowCount: sql.includes("treecrdt_meta") ? 1 : 0 }; + } + + async end(): Promise {} + } + + class MockClient { + on = vi.fn(); + + async connect(): Promise {} + + async query(sql: string): Promise<{ rowCount: number }> { + pgMockState.clientQueries.push(sql); + pgMockState.clientActiveQueries += 1; + if (pgMockState.clientActiveQueries > 1) { + pgMockState.clientActiveQueries -= 1; + throw new Error("single pg client query overlap"); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + pgMockState.clientActiveQueries -= 1; + return { rowCount: 0 }; + } + + async end(): Promise {} + } + + return { Pool: MockPool, Client: MockClient }; +}); + +beforeEach(() => { + pgMockState.clientActiveQueries = 0; + pgMockState.clientQueries.length = 0; + pgMockState.poolQueries.length = 0; +}); + +test("doc update bus does not overlap application queries on a single pg client", async () => { + const { PostgresDocUpdateBus } = await import("../src/server.ts"); + + const bus = await PostgresDocUpdateBus.create({ + postgresUrl: "postgres://example.invalid/treecrdt", + channel: "treecrdt_sync_doc_updates", + onDocUpdate: () => {}, + }); + + await expect( + Promise.all([bus.hasDoc("doc-a"), bus.ping(), bus.publishDocUpdate("doc-a")]) + ).resolves.toEqual([true, undefined, undefined]); + + expect(pgMockState.poolQueries).toHaveLength(3); + expect( + pgMockState.clientQueries.filter( + (sql) => !sql.startsWith("LISTEN ") && !sql.startsWith("UNLISTEN ") + ) + ).toEqual([]); + + await bus.close(); +}); From f846ce8ca7e5433cf4e8cd503c28efa936495ab9 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 11:04:09 +0100 Subject: [PATCH 16/58] feat(benchmark): support remote fixture reuse --- docs/BENCHMARKS.md | 29 +++++- .../scripts/bench-sync.ts | 89 ++++++++++++------- scripts/run-sync-bench.mjs | 6 +- 3 files changed, 89 insertions(+), 35 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index d71a4b09..65b09736 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -76,9 +76,9 @@ pnpm benchmark:sync:remote -- \ Use `--fanout=20` when you want to model a broader notebook tree. -### Prime Local Fixtures +### Prime Sync Server Fixtures -Use this when you want to prebuild the local Postgres server fixtures before running the actual local sync benchmarks. +Use this when you want to prebuild sync-server fixtures before running the actual sync benchmarks. ```sh pnpm benchmark:sync:prime @@ -95,13 +95,34 @@ pnpm benchmark:sync:prime -- \ --server-fixture-cache=rebuild ``` +You can also prime the remote target explicitly: + +```sh +TREECRDT_SYNC_SERVER_URL=ws://host-or-elb/sync \ +pnpm benchmark:sync:remote prime -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --server-fixture-cache=rebuild +``` + +Then benchmark against that already-seeded remote doc without reseeding: + +```sh +TREECRDT_SYNC_SERVER_URL=ws://host-or-elb/sync \ +pnpm benchmark:sync:remote -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --first-view \ + --server-fixture-cache=reuse +``` + By default, the local sync target runs the Postgres sync server in a spawned child process so local and remote measurements are closer to each other. When you add `--profile-backend`, the local target intentionally switches to the in-process server so per-backend timings are visible inside the benchmark process. Local server benchmarks now seed the Postgres backend directly before the timer starts. That keeps the measured path honest, because the actual sync to the client still goes through the real websocket server, while avoiding huge protocol-seed setup costs that are not part of the benchmark question. For read-only local server workloads, the harness now prepares that server fixture once per benchmark case and reuses it across warmup and measured samples. It also reuses the same seeded Postgres fixture across separate benchmark runs by default when the workload definition matches, so repeated `50k/100k` runs do not keep reimporting the same large server doc. -Use `--server-fixture-cache=rebuild` when you want to force a fresh local Postgres fixture, or `--server-fixture-cache=off` when you want every run to seed an isolated throwaway fixture. +Use `--server-fixture-cache=rebuild` when you want to force a fresh fixture, or `--server-fixture-cache=off` when you want every run to seed an isolated throwaway fixture. For remote fixtures, `--server-fixture-cache=reuse` assumes the deterministic fixture doc already exists and skips reseeding. ### Time To First Visible Page @@ -121,6 +142,8 @@ pnpm benchmark:sync:local -- \ For custom `--count` or `--counts` runs, the sync bench now defaults to multiple measured samples instead of silently falling back to one. Use `--iterations=N` and `--warmup=N` when you want explicit control over stability versus runtime. +Add `--post-seed-wait-ms=N` when you want to probe whether immediate post-upload backlog is skewing the measured first-view path. This is mainly a debugging aid for remote runs. + ### Small-Scope Direct Send Add `--direct-send-threshold=N` when you want to experiment with a clean-slate shortcut for small scoped syncs. diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 7ef0e4ee..44b27d24 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -91,7 +91,7 @@ type SyncBenchSample = { type PreparedServerFixture = { docId: string; cacheKey?: string; - cacheStatus: "disabled" | "hit" | "miss" | "rebuild"; + cacheStatus: "disabled" | "hit" | "miss" | "rebuild" | "assumed"; }; type SyncBenchConnection = { @@ -1526,23 +1526,37 @@ async function seedServerState( directSendThreshold: 0, ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); - const connection = await runtime.connect(docId); - const detach = peer.attach(connection.transport); - try { - await peer.syncOnce(connection.transport, { all: {} }, { - maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, - codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, - ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), - }); - await seedBackend.flush(); - if (runtime.waitForOpCount) { - await runtime.waitForOpCount(docId, { all: {} }, ops.length, { - timeoutMs: SERVER_SEED_READY_TIMEOUT_MS, + const deadline = Date.now() + SERVER_SEED_READY_TIMEOUT_MS; + let lastError: unknown; + while (true) { + const connection = await runtime.connect(docId); + const detach = peer.attach(connection.transport); + try { + await peer.syncOnce(connection.transport, { all: {} }, { + maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, + codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, + ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); + await seedBackend.flush(); + if (runtime.waitForOpCount) { + await runtime.waitForOpCount(docId, { all: {} }, ops.length, { + timeoutMs: SERVER_SEED_READY_TIMEOUT_MS, + }); + } + break; + } catch (error) { + lastError = error; + if (Date.now() >= deadline) { + throw new Error( + `timed out seeding server doc ${docId} within ${SERVER_SEED_READY_TIMEOUT_MS}ms` + + (lastError ? `: ${String(lastError)}` : "") + ); + } + await sleep(SERVER_READY_POLL_MS); + } finally { + detach(); + await connection.close(); } - } finally { - detach(); - await connection.close(); } return expectedFilterCount; } finally { @@ -1554,7 +1568,6 @@ function canReuseServerFixture( runtime: SyncBenchTargetRuntime, bench: ReturnType ): boolean { - if (runtime.id !== "local-postgres-sync-server") return false; // Reuse is safe when every client-side starting op is already present on the // server fixture, so samples only read from the shared doc and never mutate it. const serverOpIds = new Set( @@ -1667,6 +1680,13 @@ async function prepareServerFixture( // fall through to rebuild the fixture } } + if (cacheMode === "reuse" && !runtime.inspectDoc) { + return { + docId, + cacheKey, + cacheStatus: "assumed", + }; + } if (cacheMode !== "off") { await runtime.resetDoc?.(docId); } @@ -1713,6 +1733,7 @@ async function waitForServerOpCount( ): Promise { const deadline = Date.now() + timeoutMs; let lastCount = -1; + let lastError: unknown; while (true) { const verifierDb = await openDb({ storage: "memory", docId }); try { @@ -1730,6 +1751,9 @@ async function waitForServerOpCount( if (lastCount === expectedCount) { return; } + lastError = undefined; + } catch (error) { + lastError = error; } finally { verifierDb.close(); } @@ -1737,7 +1761,8 @@ async function waitForServerOpCount( if (Date.now() >= deadline) { throw new Error( `timed out waiting for server doc ${docId} to reach ${expectedCount} ops within ${timeoutMs}ms` + - (lastCount >= 0 ? ` (last count=${lastCount})` : "") + (lastCount >= 0 ? ` (last count=${lastCount})` : "") + + (lastError ? `: ${String(lastError)}` : "") ); } await sleep(SERVER_READY_POLL_MS); @@ -2212,7 +2237,7 @@ async function primeServerFixtureCase( } if (!canReuseServerFixture(runtime, bench)) { throw new Error( - `sync bench workload ${bench.name} does not support reusable local server fixtures` + `sync bench workload ${bench.name} does not support reusable sync-server fixtures` ); } @@ -2290,21 +2315,23 @@ async function main() { try { if (primeServerFixtures) { - if (targets.some((target) => target !== "local-postgres-sync-server")) { + if (targets.some((target) => target === "direct")) { throw new Error( - "--prime-server-fixtures only supports the local-postgres-sync-server target" + "--prime-server-fixtures only supports sync-server targets" ); } - const primeCases = workloads.flatMap((workload) => { - const entries = - workload === "sync-root-children-fanout10" ? rootConfig : config; - return entries.map(([size]) => ({ - target: "local-postgres-sync-server" as const, - workload, - size, - fanout, - })); - }); + const primeCases = targets.flatMap((target) => + workloads.flatMap((workload) => { + const entries = + workload === "sync-root-children-fanout10" ? rootConfig : config; + return entries.map(([size]) => ({ + target, + workload, + size, + fanout, + })); + }) + ); for (const primeCase of primeCases) { const result = await primeServerFixtureCase( diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index 17068f87..a80b9cfd 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -159,6 +159,8 @@ async function main() { pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --profile-hello pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --direct-send-threshold=64 pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --direct-send-threshold=64 --max-ops-per-batch=500 + pnpm benchmark:sync:remote prime -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --server-fixture-cache=rebuild + pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view --server-fixture-cache=reuse pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=50000 --first-view --server-fixture-cache=rebuild pnpm benchmark:sync:prime -- --counts=10000,50000,100000 TREECRDT_SYNC_SERVER_URL=wss://host/sync pnpm benchmark:sync:remote -- --workloads=sync-balanced-children-payloads-cold-start @@ -173,12 +175,14 @@ Notes: - local sync benches use a spawned child-process server by default for more realistic local vs remote comparisons - local sync benches now use a benchmark-only direct Postgres seed step before timing, so large local runs avoid spending minutes protocol-seeding data that is not part of the measured sync - local read-only sync benches reuse the same seeded Postgres fixture across warmup, samples, and later matching runs by default; use --server-fixture-cache=rebuild to refresh it or --server-fixture-cache=off to disable that cache - - pnpm benchmark:sync:prime warms local Postgres fixtures for the read-only first-view workloads and defaults to rebuilding them for 10k/50k/100k unless you override the forwarded args + - remote read-only sync benches can also reuse deterministic fixture docIds; use prime mode to seed them once, then rerun with --server-fixture-cache=reuse to benchmark steady-state first-view without reseeding + - pnpm benchmark:sync:prime warms local Postgres fixtures for the read-only first-view workloads and defaults to rebuilding them for 10k/50k/100k unless you override the forwarded args; add an explicit remote target to prime remote fixtures instead - add --profile-backend to capture listOpRefs/getOpsByOpRefs/applyOps timing per backend; on local benches this switches back to the in-process server for debug visibility - add --profile-transport to capture sync message counts, bytes, and a small event timeline - add --profile-hello to capture responder-side hello stage timings; local child-process runs parse server trace output, direct and in-process runs collect it in-process - add --direct-send-threshold=N to experiment with a clean-slate shortcut that skips the RIBLT round when the requested local filter is empty and the responder has at most N scoped ops - add --max-ops-per-batch=N to force smaller opsBatch messages when stress-testing upload paths or remote seed behavior + - add --post-seed-wait-ms=N to probe whether immediate post-upload backlog is skewing first-view timings - extra args are forwarded to packages/treecrdt-sqlite-node/scripts/bench-sync.ts`); return; } From 14397babf4a889882fb8513b64a06a5f16b96390 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 11:26:41 +0100 Subject: [PATCH 17/58] fix(sync): wait for uploaded batches to apply --- packages/sync/protocol/src/sync.ts | 70 ++++++++++++++++++---- packages/sync/protocol/tests/smoke.test.ts | 54 +++++++++++++++++ 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/packages/sync/protocol/src/sync.ts b/packages/sync/protocol/src/sync.ts index fda56a0b..57812063 100644 --- a/packages/sync/protocol/src/sync.ts +++ b/packages/sync/protocol/src/sync.ts @@ -57,6 +57,7 @@ type ResponderSession = { round: number; decoder: RibltDecoder16; expectedIndex: bigint; + awaitingIncomingDone: boolean; }; type InitiatorSession = { @@ -68,6 +69,7 @@ type InitiatorSession = { codewordCredits: number; codewordCreditSignal: Pending; receivedOps: Pending; + awaitingUploadAck: boolean; done: boolean; }; @@ -275,6 +277,7 @@ export class SyncPeer { private readonly initiatorSessions = new Map>(); private readonly responderSubscriptions = new Map>(); private readonly initiatorSubscriptions = new Map(); + private readonly opsBatchQueues = new Map>(); private pushScheduled = false; private pushRunning = false; private pushInFlight: Promise = Promise.resolve(); @@ -286,7 +289,9 @@ export class SyncPeer { opts: SyncPeerOptions = {} ) { this.maxCodewords = opts.maxCodewords ?? 50_000; - this.maxOpsPerBatch = opts.maxOpsPerBatch ?? 5_000; + // Keep wire batches modest by default; large 5k-op frames were a real + // source of remote ingest instability on production-like sync servers. + this.maxOpsPerBatch = opts.maxOpsPerBatch ?? 500; this.auth = opts.auth; this.maxHelloFilters = opts.maxHelloFilters ?? 8; this.directSendThreshold = opts.directSendThreshold ?? 0; @@ -453,6 +458,7 @@ export class SyncPeer { codewordCredits: 1, codewordCreditSignal: deferred(), receivedOps: deferred(), + awaitingUploadAck: false, done: false, }; this.initiatorSessions.set(filterId, session); @@ -794,7 +800,7 @@ export class SyncPeer { await this.onRibltStatus(msg.payload.value); return; case "opsBatch": - await this.onOpsBatch(msg.payload.value); + await this.enqueueOpsBatch(transport, msg.payload.value); return; case "subscribe": await this.onSubscribe(transport, msg.payload.value); @@ -1019,6 +1025,7 @@ export class SyncPeer { round: 0, decoder, expectedIndex: 0n, + awaitingIncomingDone: false, }); } @@ -1162,6 +1169,7 @@ export class SyncPeer { const receiverMissing = session.decoder.remoteMissing() as unknown as OpRef[]; const senderMissing = session.decoder.localMissing() as unknown as OpRef[]; const codewordsReceived = session.decoder.codewordsReceived(); + session.awaitingIncomingDone = receiverMissing.length > 0; await transport.send({ v: 0, @@ -1181,15 +1189,17 @@ export class SyncPeer { if (senderMissing.length > 0) { await this.sendOpsBatches(transport, msg.filterId, senderMissing); - } else { + if (!session.awaitingIncomingDone) { + this.responderSessions.delete(msg.filterId); + } + } else if (!session.awaitingIncomingDone) { await transport.send({ v: 0, docId: this.backend.docId, payload: { case: "opsBatch", value: { filterId: msg.filterId, ops: [], done: true } }, }); + this.responderSessions.delete(msg.filterId); } - - this.responderSessions.delete(msg.filterId); } private async onSubscribe( @@ -1361,13 +1371,39 @@ export class SyncPeer { return; } session.done = true; + if (status.payload.case === "decoded") { + session.awaitingUploadAck = status.payload.value.receiverMissing.length > 0; + } session.terminalStatus.resolve(status); const signal = session.codewordCreditSignal; session.codewordCreditSignal = deferred(); signal.resolve(); } - private async onOpsBatch(batch: OpsBatch): Promise { + private async enqueueOpsBatch( + transport: DuplexTransport>, + batch: OpsBatch + ): Promise { + const previous = this.opsBatchQueues.get(batch.filterId) ?? Promise.resolve(); + const current = previous + .catch(() => { + // A prior batch failure should not permanently poison the queue. + }) + .then(() => this.onOpsBatch(transport, batch)); + this.opsBatchQueues.set(batch.filterId, current); + try { + await current; + } finally { + if (this.opsBatchQueues.get(batch.filterId) === current) { + this.opsBatchQueues.delete(batch.filterId); + } + } + } + + private async onOpsBatch( + transport: DuplexTransport>, + batch: OpsBatch + ): Promise { const purpose: SyncOpPurpose = this.initiatorSubscriptions.has(batch.filterId) ? "subscribe" : "reconcile"; const auth = batch.auth; if (auth && auth.length !== batch.ops.length) { @@ -1423,11 +1459,25 @@ export class SyncPeer { if (allowedOps.length > 0) void this.notifyLocalUpdate(); await this.reprocessPendingOps(); - const session = this.initiatorSessions.get(batch.filterId); - if (session && batch.done) session.receivedOps.resolve(); - const responderSession = this.responderSessions.get(batch.filterId); - if (responderSession && batch.done) this.responderSessions.delete(batch.filterId); + if (responderSession && batch.done) { + if (responderSession.awaitingIncomingDone) { + responderSession.awaitingIncomingDone = false; + await transport.send({ + v: 0, + docId: this.backend.docId, + payload: { case: "opsBatch", value: { filterId: batch.filterId, ops: [], done: true } }, + }); + } + this.responderSessions.delete(batch.filterId); + } + + const session = this.initiatorSessions.get(batch.filterId); + if (session && batch.done) { + if (!session.awaitingUploadAck || batch.ops.length === 0) { + session.receivedOps.resolve(); + } + } } private async reprocessPendingOps(): Promise { diff --git a/packages/sync/protocol/tests/smoke.test.ts b/packages/sync/protocol/tests/smoke.test.ts index e121e74f..fa7fc5a6 100644 --- a/packages/sync/protocol/tests/smoke.test.ts +++ b/packages/sync/protocol/tests/smoke.test.ts @@ -309,6 +309,17 @@ class FailingApplyBackend extends MemoryBackend { } } +class DelayedApplyBackend extends MemoryBackend { + constructor(docId: string, private readonly delayMs: number) { + super(docId); + } + + override async applyOps(ops: Operation[]): Promise { + await new Promise((resolve) => setTimeout(resolve, this.delayMs)); + await super.applyOps(ops); + } +} + test("syncOnce does not starve macrotask transports", async () => { const docId = "doc-sync-macrotask"; const root = "0".repeat(32); @@ -367,6 +378,49 @@ test("syncOnce paces outbound codewords until delayed ribltStatus arrives", asyn await waitUntil(() => b.hasOp(replicaHex.a, 1), { message: "expected b to receive a:1 after delayed ribltStatus" }); }); +test("syncOnce waits for responder to apply uploaded ops before resolving", async () => { + const docId = "doc-sync-upload-ack"; + const root = "0".repeat(32); + + const a = new MemoryBackend(docId); + const b = new DelayedApplyBackend(docId, 20); + + await a.applyOps([ + makeOp(replicas.a, 1, 1, { + type: "insert", + parent: root, + node: nodeIdFromInt(1), + orderKey: orderKeyFromPosition(0), + }), + makeOp(replicas.a, 2, 2, { + type: "insert", + parent: root, + node: nodeIdFromInt(2), + orderKey: orderKeyFromPosition(1), + }), + makeOp(replicas.a, 3, 3, { + type: "insert", + parent: root, + node: nodeIdFromInt(3), + orderKey: orderKeyFromPosition(2), + }), + ]); + + const [wa, wb] = createMacrotaskDuplex(); + const ta = wrapDuplexTransportWithCodec(wa, treecrdtSyncV0ProtobufCodec); + const tb = wrapDuplexTransportWithCodec(wb, treecrdtSyncV0ProtobufCodec); + const pa = new SyncPeer(a); + const pb = new SyncPeer(b); + pa.attach(ta); + pb.attach(tb); + + await pa.syncOnce(ta, { all: {} }, { maxCodewords: 10_000, codewordsPerMessage: 256, maxOpsPerBatch: 1 }); + + expect(b.hasOp(replicaHex.a, 1)).toBe(true); + expect(b.hasOp(replicaHex.a, 2)).toBe(true); + expect(b.hasOp(replicaHex.a, 3)).toBe(true); +}); + test("syncOnce protobuf roundtrips ribltStatus.more", () => { const msg = { v: 0, From 6dfa2b8b92e2bb27dcf6d8680104ab87e62c0379 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 11:42:17 +0100 Subject: [PATCH 18/58] feat(benchmark): add explicit sync upload benchmark --- docs/BENCHMARKS.md | 19 ++++++++++ package.json | 3 ++ .../scripts/bench-sync.ts | 35 ++++++++++++++----- scripts/run-sync-bench.mjs | 3 ++ 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 65b09736..d1c4973a 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -144,6 +144,25 @@ For custom `--count` or `--counts` runs, the sync bench now defaults to multiple Add `--post-seed-wait-ms=N` when you want to probe whether immediate post-upload backlog is skewing the measured first-view path. This is mainly a debugging aid for remote runs. +### Upload Benchmarks + +Use prime/upload mode when you want an explicit benchmark for seeding a sync-server doc. + +This measures the full server-fixture creation path and writes a result file under `benchmarks/sqlite-node-sync/server-fixture-*.json` with `durationMs`, `opsPerSec`, and the seeded `fixtureOpCount`. + +```sh +TREECRDT_SYNC_SERVER_URL=ws://host/sync \ +pnpm benchmark:sync:upload:remote -- \ + --workloads=sync-balanced-children-payloads-cold-start \ + --count=10000 \ + --server-fixture-cache=rebuild +``` + +Use this to answer a different question than first-view: + +- `benchmark:sync:remote ... --first-view` answers "how fast can a new device open an existing subtree?" +- `benchmark:sync:upload:remote ...` answers "how long does it take to upload and materialize a large tree on the sync server?" + ### Small-Scope Direct Send Add `--direct-send-threshold=N` when you want to experiment with a clean-slate shortcut for small scoped syncs. diff --git a/package.json b/package.json index 127e8c1c..fc5efce3 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,9 @@ "benchmark:sync:direct": "node scripts/run-sync-bench.mjs direct", "benchmark:sync:local": "node scripts/run-sync-bench.mjs local", "benchmark:sync:prime": "node scripts/run-sync-bench.mjs prime", + "benchmark:sync:upload": "node scripts/run-sync-bench.mjs prime", + "benchmark:sync:upload:local": "node scripts/run-sync-bench.mjs local prime", + "benchmark:sync:upload:remote": "node scripts/run-sync-bench.mjs remote prime", "benchmark:sync:remote": "node scripts/run-sync-bench.mjs remote", "benchmark:wasm": "pnpm -C packages/treecrdt-wasm-js run benchmark", "benchmark:web": "pnpm -C packages/treecrdt-wa-sqlite/e2e run bench", diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index 44b27d24..a47333c8 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -76,6 +76,9 @@ type SyncBenchResult = { type PrimedServerFixtureResult = { name: string; + totalOps: number; + durationMs: number; + opsPerSec: number; extra?: Record; }; @@ -2241,6 +2244,7 @@ async function primeServerFixtureCase( ); } + const startedAt = performance.now(); const preparedFixture = await prepareServerFixture( runtime, bench, @@ -2248,8 +2252,14 @@ async function primeServerFixtureCase( serverFixtureCacheMode, maxOpsPerBatch ); + const durationMs = performance.now() - startedAt; + const totalOps = bench.opsB.length; + const opsPerSec = durationMs > 0 ? (totalOps / durationMs) * 1000 : Infinity; return { name: `${bench.name}-server-fixture`, + totalOps, + durationMs, + opsPerSec, extra: { ...bench.extra, count: benchCase.size, @@ -2271,7 +2281,7 @@ async function primeServerFixtureCase( serverFixtureCacheStatus: preparedFixture.cacheStatus, serverFixtureCacheKey: preparedFixture.cacheKey, docId: preparedFixture.docId, - fixtureOpCount: bench.opsB.length, + fixtureOpCount: totalOps, fixtureMaxLamport: maxLamport(bench.opsB), }, }; @@ -2341,14 +2351,23 @@ async function main() { serverFixtureCacheMode, maxOpsPerBatch ); - console.log( - JSON.stringify({ - implementation: "sqlite-node", - workload: result.name, - target: primeCase.target, - extra: result.extra, - }) + const outFile = path.join( + repoRoot, + "benchmarks", + "sqlite-node-sync", + `server-fixture-${primeCase.target}-${result.name}.json` ); + const payload = await writeResult(result, { + implementation: "sqlite-node", + storage: "server-fixture", + workload: result.name, + outFile, + extra: { + target: primeCase.target, + ...result.extra, + }, + }); + console.log(JSON.stringify(payload)); } return; } diff --git a/scripts/run-sync-bench.mjs b/scripts/run-sync-bench.mjs index a80b9cfd..7e235870 100644 --- a/scripts/run-sync-bench.mjs +++ b/scripts/run-sync-bench.mjs @@ -150,6 +150,8 @@ async function main() { console.log(`Usage: pnpm benchmark:sync pnpm benchmark:sync:prime + pnpm benchmark:sync:upload + pnpm benchmark:sync:upload:remote -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 pnpm benchmark:sync:direct -- --workloads=sync-balanced-children-payloads-cold-start --counts=10000,50000 --fanout=10 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-cold-start --count=10000 pnpm benchmark:sync:local -- --workloads=sync-balanced-children-payloads-cold-start --count=10000 --first-view @@ -176,6 +178,7 @@ Notes: - local sync benches now use a benchmark-only direct Postgres seed step before timing, so large local runs avoid spending minutes protocol-seeding data that is not part of the measured sync - local read-only sync benches reuse the same seeded Postgres fixture across warmup, samples, and later matching runs by default; use --server-fixture-cache=rebuild to refresh it or --server-fixture-cache=off to disable that cache - remote read-only sync benches can also reuse deterministic fixture docIds; use prime mode to seed them once, then rerun with --server-fixture-cache=reuse to benchmark steady-state first-view without reseeding + - prime/upload mode now records a real durationMs/opsPerSec result and writes a benchmark JSON file under benchmarks/sqlite-node-sync/server-fixture-*.json - pnpm benchmark:sync:prime warms local Postgres fixtures for the read-only first-view workloads and defaults to rebuilding them for 10k/50k/100k unless you override the forwarded args; add an explicit remote target to prime remote fixtures instead - add --profile-backend to capture listOpRefs/getOpsByOpRefs/applyOps timing per backend; on local benches this switches back to the in-process server for debug visibility - add --profile-transport to capture sync message counts, bytes, and a small event timeline From 008c8f53452da34f9566eef14e6b7ee5cad51ec2 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 11:55:03 +0100 Subject: [PATCH 19/58] feat(sync): shortcut uploads to empty receivers --- packages/sync/protocol/src/sync.ts | 63 ++++++++++++++++++++++ packages/sync/protocol/tests/smoke.test.ts | 47 ++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/packages/sync/protocol/src/sync.ts b/packages/sync/protocol/src/sync.ts index 57812063..d94d9318 100644 --- a/packages/sync/protocol/src/sync.ts +++ b/packages/sync/protocol/src/sync.ts @@ -51,6 +51,8 @@ function deferred(): Pending { const DIRECT_SEND_SMALL_SCOPE_SUPPORT_CAPABILITY = "treecrdt.sync.direct_send_small_scope.v1"; const DIRECT_SEND_SMALL_SCOPE_REQUEST_CAPABILITY = "treecrdt.sync.direct_send_small_scope.request.v1"; const DIRECT_SEND_SMALL_SCOPE_FILTER_CAPABILITY = "treecrdt.sync.direct_send_small_scope.filter.v1"; +const DIRECT_SEND_EMPTY_RECEIVER_SUPPORT_CAPABILITY = "treecrdt.sync.direct_send_empty_receiver.v1"; +const DIRECT_SEND_EMPTY_RECEIVER_FILTER_CAPABILITY = "treecrdt.sync.direct_send_empty_receiver.filter.v1"; type ResponderSession = { filter: Filter; @@ -264,6 +266,25 @@ function peerRequestedDirectSendFilter( ); } +function peerSupportsDirectSendEmptyReceiver(capabilities: readonly Capability[]): boolean { + return capabilities.some( + (capability) => + capability.name === DIRECT_SEND_EMPTY_RECEIVER_SUPPORT_CAPABILITY && + capability.value === "1" + ); +} + +function peerSelectedDirectSendEmptyReceiverFilter( + capabilities: readonly Capability[], + filterId: string +): boolean { + return capabilities.some( + (capability) => + capability.name === DIRECT_SEND_EMPTY_RECEIVER_FILTER_CAPABILITY && + capability.value === filterId + ); +} + export class SyncPeer { private readonly maxCodewords: number; private readonly maxOpsPerBatch: number; @@ -277,6 +298,7 @@ export class SyncPeer { private readonly initiatorSessions = new Map>(); private readonly responderSubscriptions = new Map>(); private readonly initiatorSubscriptions = new Map(); + private readonly responderAwaitingUploadAcks = new Set(); private readonly opsBatchQueues = new Map>(); private pushScheduled = false; private pushRunning = false; @@ -438,6 +460,12 @@ export class SyncPeer { value: "1", }); } + if (!peerSupportsDirectSendEmptyReceiver(capabilities)) { + capabilities.push({ + name: DIRECT_SEND_EMPTY_RECEIVER_SUPPORT_CAPABILITY, + value: "1", + }); + } if ( localOpRefsBeforeHello.length === 0 && !peerRequestedDirectSendFilter(capabilities, filterId) @@ -498,6 +526,24 @@ export class SyncPeer { opRefs = opRefs.filter((_r, idx) => allowed[idx] === true); } + if (peerSelectedDirectSendEmptyReceiverFilter(ack.capabilities, filterId)) { + session.awaitingUploadAck = true; + if (opRefs.length > 0) { + await this.sendOpsBatches(transport, filterId, opRefs, { + maxOpsPerBatch: opts.maxOpsPerBatch, + filter, + }); + } else { + await transport.send({ + v: 0, + docId: this.backend.docId, + payload: { case: "opsBatch", value: { filterId, ops: [], done: true } }, + }); + } + await session.receivedOps.promise; + return; + } + const enc = new RibltEncoder16(); for (const r of opRefs) enc.addSymbol(r); @@ -1014,6 +1060,15 @@ export class SyncPeer { continue; } + if (peerSupportsDirectSendEmptyReceiver(hello.capabilities) && localOpRefs.length === 0) { + ackCapabilities.push({ + name: DIRECT_SEND_EMPTY_RECEIVER_FILTER_CAPABILITY, + value: id, + }); + this.responderAwaitingUploadAcks.add(id); + continue; + } + const decoder = new RibltDecoder16(); for (const r of localOpRefs) decoder.addLocalSymbol(r); traceHello(this.backend.docId, traceStartedAt, "after-decoder-setup", { @@ -1460,6 +1515,14 @@ export class SyncPeer { await this.reprocessPendingOps(); const responderSession = this.responderSessions.get(batch.filterId); + if (!responderSession && this.responderAwaitingUploadAcks.has(batch.filterId) && batch.done) { + this.responderAwaitingUploadAcks.delete(batch.filterId); + await transport.send({ + v: 0, + docId: this.backend.docId, + payload: { case: "opsBatch", value: { filterId: batch.filterId, ops: [], done: true } }, + }); + } if (responderSession && batch.done) { if (responderSession.awaitingIncomingDone) { responderSession.awaitingIncomingDone = false; diff --git a/packages/sync/protocol/tests/smoke.test.ts b/packages/sync/protocol/tests/smoke.test.ts index fa7fc5a6..14cafd34 100644 --- a/packages/sync/protocol/tests/smoke.test.ts +++ b/packages/sync/protocol/tests/smoke.test.ts @@ -528,6 +528,53 @@ test("syncOnce can direct-send a small clean-slate scope without riblt codewords expect(wire).not.toContain("bToA:ribltStatus"); }); +test("syncOnce can direct-send a clean-slate upload to an empty receiver without riblt codewords", async () => { + const docId = "doc-sync-direct-send-empty-receiver"; + const root = "0".repeat(32); + + const a = new MemoryBackend(docId); + const b = new MemoryBackend(docId); + + await a.applyOps([ + makeOp(replicas.a, 1, 1, { + type: "insert", + parent: root, + node: nodeIdFromInt(1), + orderKey: orderKeyFromPosition(0), + }), + makeOp(replicas.a, 2, 2, { + type: "insert", + parent: root, + node: nodeIdFromInt(2), + orderKey: orderKeyFromPosition(1), + }), + ]); + + const [ta, tb, log] = createLoggedTimedDuplex>(); + const pa = new SyncPeer(a); + const pb = new SyncPeer(b); + pa.attach(ta); + pb.attach(tb); + + await pa.syncOnce(ta, { all: {} }, { + maxCodewords: 4_096, + codewordsPerMessage: 16, + maxOpsPerBatch: 1, + }); + + await waitUntil(() => b.hasOp(replicaHex.a, 2), { + message: "expected empty receiver to receive direct-sent upload", + }); + + const wire = log.map((entry) => `${entry.dir}:${entry.msg.payload.case}`); + expect(wire).toContain("aToB:hello"); + expect(wire).toContain("bToA:helloAck"); + expect(wire).toContain("aToB:opsBatch"); + expect(wire).toContain("bToA:opsBatch"); + expect(wire).not.toContain("aToB:ribltCodewords"); + expect(wire).not.toContain("bToA:ribltStatus"); +}); + test("syncOnce rejects when local message handler throws during apply", async () => { const docId = "doc-sync-apply-error"; const root = "0".repeat(32); From f74853f273441c11eab6e841a767ef22f4740909 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 12:53:50 +0100 Subject: [PATCH 20/58] fix(benchmark): use fresh remote docs for rebuild --- packages/treecrdt-sqlite-node/scripts/bench-sync.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index a47333c8..a438540c 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -1662,10 +1662,13 @@ async function prepareServerFixture( ): Promise { const cacheKey = cacheMode === "off" ? undefined : createServerFixtureCacheKey(bench); + const hasResettableFixture = runtime.resetDoc != null; const docId = cacheMode === "off" ? `sqlite-node-sync-bench-${runtime.id}-fixture-${crypto.randomUUID()}` - : `sqlite-node-sync-bench-${runtime.id}-fixture-${cacheKey}`; + : cacheMode === "rebuild" && !hasResettableFixture + ? `sqlite-node-sync-bench-${runtime.id}-fixture-${cacheKey}-${crypto.randomUUID()}` + : `sqlite-node-sync-bench-${runtime.id}-fixture-${cacheKey}`; if (cacheMode === "reuse" && runtime.inspectDoc) { try { const current = await runtime.inspectDoc(docId); From 2fe88567eead2b611ec4e429ef7b5045aff7a0b7 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 17:30:03 +0100 Subject: [PATCH 21/58] fix(sync): defer large postgres upload materialization --- packages/sync/protocol/src/sync.ts | 5 ++- packages/treecrdt-postgres-rs/src/store.rs | 9 ++++- .../tests/postgres_test.rs | 40 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/packages/sync/protocol/src/sync.ts b/packages/sync/protocol/src/sync.ts index d94d9318..9a5c7074 100644 --- a/packages/sync/protocol/src/sync.ts +++ b/packages/sync/protocol/src/sync.ts @@ -53,6 +53,7 @@ const DIRECT_SEND_SMALL_SCOPE_REQUEST_CAPABILITY = "treecrdt.sync.direct_send_sm const DIRECT_SEND_SMALL_SCOPE_FILTER_CAPABILITY = "treecrdt.sync.direct_send_small_scope.filter.v1"; const DIRECT_SEND_EMPTY_RECEIVER_SUPPORT_CAPABILITY = "treecrdt.sync.direct_send_empty_receiver.v1"; const DIRECT_SEND_EMPTY_RECEIVER_FILTER_CAPABILITY = "treecrdt.sync.direct_send_empty_receiver.filter.v1"; +const DIRECT_SEND_EMPTY_RECEIVER_MAX_OPS_PER_BATCH = 5_000; type ResponderSession = { filter: Filter; @@ -528,9 +529,11 @@ export class SyncPeer { if (peerSelectedDirectSendEmptyReceiverFilter(ack.capabilities, filterId)) { session.awaitingUploadAck = true; + const uploadMaxOpsPerBatch = + opts.maxOpsPerBatch ?? DIRECT_SEND_EMPTY_RECEIVER_MAX_OPS_PER_BATCH; if (opRefs.length > 0) { await this.sendOpsBatches(transport, filterId, opRefs, { - maxOpsPerBatch: opts.maxOpsPerBatch, + maxOpsPerBatch: uploadMaxOpsPerBatch, filter, }); } else { diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index 3c8b8221..8d9fcf93 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -13,6 +13,8 @@ use treecrdt_core::{ use crate::opref::{derive_op_ref_v0, OPREF_V0_WIDTH}; +const DEFER_INCREMENTAL_MATERIALIZATION_MIN_OPS: usize = 2_000; + fn storage_debug(e: E) -> Error { Error::Storage(format!("{e:?}")) } @@ -1115,8 +1117,13 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation } let inserted = inserted_ops.len(); + if meta.dirty || inserted >= DEFER_INCREMENTAL_MATERIALIZATION_MIN_OPS { + set_tree_meta_dirty(client, doc_id, true)?; + return Ok(inserted.min(u64::MAX as usize) as u64); + } + let _ = try_incremental_materialization( - meta.dirty, + false, || materialize_ops_in_order(ctx, &meta, inserted_ops), || { let _ = set_tree_meta_dirty(client, doc_id, true); diff --git a/packages/treecrdt-postgres-rs/tests/postgres_test.rs b/packages/treecrdt-postgres-rs/tests/postgres_test.rs index e24c592e..d40c81e2 100644 --- a/packages/treecrdt-postgres-rs/tests/postgres_test.rs +++ b/packages/treecrdt-postgres-rs/tests/postgres_test.rs @@ -67,6 +67,46 @@ fn postgres_backend_apply_is_idempotent_and_max_lamport_monotonic() { assert_eq!(max, 7); } +#[test] +fn postgres_backend_large_append_rebuilds_materialized_views_on_demand() { + let Some(client) = connect() else { + return; + }; + ensure_schema_once(&client); + + let doc_id = format!("test-{}", Uuid::new_v4()); + { + let mut c = client.borrow_mut(); + reset_doc_for_tests(&mut c, &doc_id).unwrap(); + } + + let replica = ReplicaId::new(b"bulk"); + let op_count = 2_500u64; + let ops: Vec = (0..op_count) + .map(|index| { + Operation::insert( + &replica, + index + 1, + index + 1, + NodeId::ROOT, + node(10_000 + index as u128), + order_key_from_position(index as u16), + ) + }) + .collect(); + + let inserted = append_ops(&client, &doc_id, &ops).unwrap(); + assert_eq!(inserted, op_count); + assert_eq!(list_op_refs_all(&client, &doc_id).unwrap().len(), op_count as usize); + assert_eq!(max_lamport(&client, &doc_id).unwrap(), op_count); + + let children = tree_children(&client, &doc_id, NodeId::ROOT).unwrap(); + assert_eq!(children.len(), op_count as usize); + + let refs_root = list_op_refs_children(&client, &doc_id, NodeId::ROOT).unwrap(); + assert_eq!(refs_root.len(), op_count as usize); +} + #[test] fn postgres_backend_doc_isolation() { let Some(client) = connect() else { From 86e5758e2f29e22c4ce471cc13bca47c00dccac5 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 18:10:15 +0100 Subject: [PATCH 22/58] perf(postgres): bulk insert deferred sync uploads --- packages/treecrdt-postgres-rs/src/store.rs | 99 ++++++++++++++++++++-- 1 file changed, 94 insertions(+), 5 deletions(-) diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index 8d9fcf93..332267a0 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -1044,6 +1044,88 @@ fn insert_op_in_tx(ctx: &PgCtx, c: &mut Client, op: &Operation) -> Result Ok(inserted > 0) } +fn bulk_insert_ops_in_tx(ctx: &PgCtx, c: &mut Client, ops: &[Operation]) -> Result { + if ops.is_empty() { + return Ok(0); + } + + let mut op_refs: Vec> = Vec::with_capacity(ops.len()); + let mut lamports: Vec = Vec::with_capacity(ops.len()); + let mut replicas: Vec> = Vec::with_capacity(ops.len()); + let mut counters: Vec = Vec::with_capacity(ops.len()); + let mut kinds: Vec = Vec::with_capacity(ops.len()); + let mut parents: Vec>> = Vec::with_capacity(ops.len()); + let mut nodes: Vec> = Vec::with_capacity(ops.len()); + let mut new_parents: Vec>> = Vec::with_capacity(ops.len()); + let mut order_keys: Vec>> = Vec::with_capacity(ops.len()); + let mut payloads: Vec>> = Vec::with_capacity(ops.len()); + let mut known_states: Vec>> = Vec::with_capacity(ops.len()); + + for op in ops { + let replica = op.meta.id.replica.as_bytes(); + let counter = op.meta.id.counter; + let op_ref = derive_op_ref_v0(&ctx.doc_id, replica, counter); + let row = op_kind_to_db(op)?; + + op_refs.push(op_ref.to_vec()); + lamports.push(op.meta.lamport as i64); + replicas.push(replica.to_vec()); + counters.push(counter as i64); + kinds.push(row.kind.to_string()); + parents.push(row.parent); + nodes.push(row.node); + new_parents.push(row.new_parent); + order_keys.push(row.order_key); + payloads.push(row.payload); + known_states.push(row.known_state); + } + + let stmt = ctx.stmt( + c, + "INSERT INTO treecrdt_ops (doc_id, op_ref, lamport, replica, counter, kind, parent, node, new_parent, order_key, payload, known_state) \ + SELECT \ + $1, \ + src.op_ref, src.lamport, src.replica, src.counter, src.kind, src.parent, src.node, src.new_parent, src.order_key, src.payload, src.known_state \ + FROM unnest( \ + $2::bytea[], \ + $3::bigint[], \ + $4::bytea[], \ + $5::bigint[], \ + $6::text[], \ + $7::bytea[], \ + $8::bytea[], \ + $9::bytea[], \ + $10::bytea[], \ + $11::bytea[], \ + $12::bytea[] \ + ) AS src(op_ref, lamport, replica, counter, kind, parent, node, new_parent, order_key, payload, known_state) \ + ON CONFLICT (doc_id, op_ref) DO NOTHING \ + RETURNING 1", + )?; + + let rows = c + .query( + &stmt, + &[ + &ctx.doc_id, + &op_refs, + &lamports, + &replicas, + &counters, + &kinds, + &parents, + &nodes, + &new_parents, + &order_keys, + &payloads, + &known_states, + ], + ) + .map_err(storage_debug)?; + + Ok(rows.len().min(u64::MAX as usize) as u64) +} + fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> Result<()> { if ops.is_empty() { return Ok(()); @@ -1101,6 +1183,18 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation // derived tables + head_seq and is not safe to run concurrently for the same doc_id). let meta = load_tree_meta_for_update(client, doc_id)?; let ctx = PgCtx::new(client.clone(), doc_id)?; + let defer_incremental = meta.dirty || ops.len() >= DEFER_INCREMENTAL_MATERIALIZATION_MIN_OPS; + + if defer_incremental { + let inserted = { + let mut c = client.borrow_mut(); + bulk_insert_ops_in_tx(&ctx, &mut c, ops)? + }; + if inserted > 0 { + set_tree_meta_dirty(client, doc_id, true)?; + } + return Ok(inserted); + } let mut inserted_ops: Vec = Vec::new(); { @@ -1117,11 +1211,6 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation } let inserted = inserted_ops.len(); - if meta.dirty || inserted >= DEFER_INCREMENTAL_MATERIALIZATION_MIN_OPS { - set_tree_meta_dirty(client, doc_id, true)?; - return Ok(inserted.min(u64::MAX as usize) as u64); - } - let _ = try_incremental_materialization( false, || materialize_ops_in_order(ctx, &meta, inserted_ops), From a6d75ae3bb75d053975c93ea4c57f3bb810f9c5a Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 18:59:36 +0100 Subject: [PATCH 23/58] perf(postgres): bulk insert incremental sync uploads --- packages/treecrdt-postgres-rs/src/store.rs | 50 ++++++++++------ .../scripts/bench-sync.ts | 57 ++++++++++++++++--- 2 files changed, 81 insertions(+), 26 deletions(-) diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index 332267a0..8793514c 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -1,5 +1,5 @@ use std::cell::RefCell; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; use postgres::{Client, Row, Statement}; @@ -13,8 +13,6 @@ use treecrdt_core::{ use crate::opref::{derive_op_ref_v0, OPREF_V0_WIDTH}; -const DEFER_INCREMENTAL_MATERIALIZATION_MIN_OPS: usize = 2_000; - fn storage_debug(e: E) -> Error { Error::Storage(format!("{e:?}")) } @@ -1044,9 +1042,9 @@ fn insert_op_in_tx(ctx: &PgCtx, c: &mut Client, op: &Operation) -> Result Ok(inserted > 0) } -fn bulk_insert_ops_in_tx(ctx: &PgCtx, c: &mut Client, ops: &[Operation]) -> Result { +fn bulk_insert_ops_in_tx(ctx: &PgCtx, c: &mut Client, ops: &[Operation]) -> Result>> { if ops.is_empty() { - return Ok(0); + return Ok(Vec::new()); } let mut op_refs: Vec> = Vec::with_capacity(ops.len()); @@ -1100,7 +1098,7 @@ fn bulk_insert_ops_in_tx(ctx: &PgCtx, c: &mut Client, ops: &[Operation]) -> Resu $12::bytea[] \ ) AS src(op_ref, lamport, replica, counter, kind, parent, node, new_parent, order_key, payload, known_state) \ ON CONFLICT (doc_id, op_ref) DO NOTHING \ - RETURNING 1", + RETURNING op_ref", )?; let rows = c @@ -1123,7 +1121,11 @@ fn bulk_insert_ops_in_tx(ctx: &PgCtx, c: &mut Client, ops: &[Operation]) -> Resu ) .map_err(storage_debug)?; - Ok(rows.len().min(u64::MAX as usize) as u64) + let mut inserted = Vec::with_capacity(rows.len()); + for row in rows { + inserted.push(row.get::<_, Vec>(0)); + } + Ok(inserted) } fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> Result<()> { @@ -1183,29 +1185,41 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation // derived tables + head_seq and is not safe to run concurrently for the same doc_id). let meta = load_tree_meta_for_update(client, doc_id)?; let ctx = PgCtx::new(client.clone(), doc_id)?; - let defer_incremental = meta.dirty || ops.len() >= DEFER_INCREMENTAL_MATERIALIZATION_MIN_OPS; - if defer_incremental { + if meta.dirty { let inserted = { let mut c = client.borrow_mut(); bulk_insert_ops_in_tx(&ctx, &mut c, ops)? }; - if inserted > 0 { + if !inserted.is_empty() { set_tree_meta_dirty(client, doc_id, true)?; } - return Ok(inserted); + return Ok(inserted.len().min(u64::MAX as usize) as u64); } - let mut inserted_ops: Vec = Vec::new(); - { + let inserted_op_refs = { let mut c = client.borrow_mut(); - for op in ops { - if insert_op_in_tx(&ctx, &mut c, op)? { - inserted_ops.push(op.clone()); - } - } + bulk_insert_ops_in_tx(&ctx, &mut c, ops)? + }; + + if inserted_op_refs.is_empty() { + return Ok(0); } + let inserted_ops: Vec = if inserted_op_refs.len() == ops.len() { + ops.to_vec() + } else { + let inserted_op_refs: HashSet> = inserted_op_refs.into_iter().collect(); + ops.iter() + .filter(|op| { + let replica = op.meta.id.replica.as_bytes(); + let counter = op.meta.id.counter; + inserted_op_refs.contains(&derive_op_ref_v0(&ctx.doc_id, replica, counter).to_vec()) + }) + .cloned() + .collect() + }; + if inserted_ops.is_empty() { return Ok(0); } diff --git a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts index a438540c..758639ee 100644 --- a/packages/treecrdt-sqlite-node/scripts/bench-sync.ts +++ b/packages/treecrdt-sqlite-node/scripts/bench-sync.ts @@ -82,6 +82,12 @@ type PrimedServerFixtureResult = { extra?: Record; }; +type SeedServerStateResult = { + expectedFilterCount: number; + uploadMs: number; + allReadyMs: number; +}; + type SyncBenchSample = { totalMs: number; syncMs: number; @@ -95,6 +101,10 @@ type PreparedServerFixture = { docId: string; cacheKey?: string; cacheStatus: "disabled" | "hit" | "miss" | "rebuild" | "assumed"; + seedUploadMs?: number; + seedAllReadyMs?: number; + filterReadyMs?: number; + totalPrepareMs?: number; }; type SyncBenchConnection = { @@ -1503,8 +1513,14 @@ async function seedServerState( ops: Operation[], filter: Filter, maxOpsPerBatch?: number -): Promise { - if (ops.length === 0) return 0; +): Promise { + if (ops.length === 0) { + return { + expectedFilterCount: 0, + uploadMs: 0, + allReadyMs: 0, + }; + } const seedDb = await openDb({ storage: "memory", docId }); try { @@ -1516,13 +1532,20 @@ async function seedServerState( }); const expectedFilterCount = (await seedBackend.listOpRefs(filter)).length; if (runtime.seedOps) { + const uploadStartedAt = performance.now(); await runtime.seedOps(docId, ops); + const uploadMs = performance.now() - uploadStartedAt; + const allReadyStartedAt = performance.now(); if (runtime.waitForOpCount) { await runtime.waitForOpCount(docId, { all: {} }, ops.length, { timeoutMs: SERVER_SEED_READY_TIMEOUT_MS, }); } - return expectedFilterCount; + return { + expectedFilterCount, + uploadMs, + allReadyMs: performance.now() - allReadyStartedAt, + }; } const peer = new SyncPeer(seedBackend, { maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, @@ -1535,18 +1558,25 @@ async function seedServerState( const connection = await runtime.connect(docId); const detach = peer.attach(connection.transport); try { + const uploadStartedAt = performance.now(); await peer.syncOnce(connection.transport, { all: {} }, { maxCodewords: SYNC_BENCH_SEED_MAX_CODEWORDS, codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE, ...(maxOpsPerBatch != null ? { maxOpsPerBatch } : {}), }); await seedBackend.flush(); + const uploadMs = performance.now() - uploadStartedAt; + const allReadyStartedAt = performance.now(); if (runtime.waitForOpCount) { await runtime.waitForOpCount(docId, { all: {} }, ops.length, { timeoutMs: SERVER_SEED_READY_TIMEOUT_MS, }); } - break; + return { + expectedFilterCount, + uploadMs, + allReadyMs: performance.now() - allReadyStartedAt, + }; } catch (error) { lastError = error; if (Date.now() >= deadline) { @@ -1561,7 +1591,7 @@ async function seedServerState( await connection.close(); } } - return expectedFilterCount; + throw new Error(`failed to seed server doc ${docId}: unexpected retry loop exit`); } finally { seedDb.close(); } @@ -1660,6 +1690,7 @@ async function prepareServerFixture( cacheMode: ServerFixtureCacheMode, maxOpsPerBatch?: number ): Promise { + const prepareStartedAt = performance.now(); const cacheKey = cacheMode === "off" ? undefined : createServerFixtureCacheKey(bench); const hasResettableFixture = runtime.resetDoc != null; @@ -1696,18 +1727,19 @@ async function prepareServerFixture( if (cacheMode !== "off") { await runtime.resetDoc?.(docId); } - const expectedFilterCount = await seedServerState( + const seedState = await seedServerState( runtime, docId, bench.opsB, bench.filter as Filter, maxOpsPerBatch ); + const filterReadyStartedAt = performance.now(); if (runtime.waitForOpCount) { await runtime.waitForOpCount( docId, bench.filter as Filter, - expectedFilterCount, + seedState.expectedFilterCount, { timeoutMs: SERVER_READY_TIMEOUT_MS } ); } else { @@ -1715,16 +1747,21 @@ async function prepareServerFixture( runtime, docId, bench.filter as Filter, - expectedFilterCount, + seedState.expectedFilterCount, directSendThreshold, maxOpsPerBatch, SERVER_SEED_READY_TIMEOUT_MS ); } + const filterReadyMs = performance.now() - filterReadyStartedAt; return { docId, cacheKey, cacheStatus: cacheMode === "rebuild" ? "rebuild" : cacheMode === "reuse" ? "miss" : "disabled", + seedUploadMs: seedState.uploadMs, + seedAllReadyMs: seedState.allReadyMs, + filterReadyMs, + totalPrepareMs: performance.now() - prepareStartedAt, }; } @@ -2284,6 +2321,10 @@ async function primeServerFixtureCase( serverFixtureCacheStatus: preparedFixture.cacheStatus, serverFixtureCacheKey: preparedFixture.cacheKey, docId: preparedFixture.docId, + seedUploadMs: preparedFixture.seedUploadMs, + seedAllReadyMs: preparedFixture.seedAllReadyMs, + filterReadyMs: preparedFixture.filterReadyMs, + totalPrepareMs: preparedFixture.totalPrepareMs, fixtureOpCount: totalOps, fixtureMaxLamport: maxLamport(bench.opsB), }, From e2c9204a2df6b94c6f28e44651a6044dc40a59e1 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 19:40:43 +0100 Subject: [PATCH 24/58] feat(debug): profile postgres sync upload phases --- packages/treecrdt-postgres-rs/src/store.rs | 237 ++++++++++++++++++++- 1 file changed, 235 insertions(+), 2 deletions(-) diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index 8793514c..301e35ba 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -1,6 +1,8 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::rc::Rc; +use std::sync::OnceLock; +use std::time::Instant; use postgres::{Client, Row, Statement}; @@ -47,6 +49,100 @@ fn vv_from_bytes(bytes: &[u8]) -> Result { serde_json::from_slice(bytes).map_err(|e| Error::Storage(e.to_string())) } +fn append_profile_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var("TREECRDT_PG_PROFILE_UPLOAD").ok().as_deref(), + Some("1") | Some("true") | Some("TRUE") | Some("yes") | Some("YES") + ) + }) +} + +#[derive(Clone, Debug, Default)] +struct PgAppendProfile { + batch_ops: usize, + doc_dirty_before: bool, + head_seq_before: u64, + bulk_insert_ms: f64, + bulk_inserted_ops: usize, + dedupe_filter_ms: f64, + materialize_ms: f64, + update_head_ms: f64, + fallback_mark_dirty: bool, + node_load_count: u64, + node_load_ms: f64, + node_ensure_count: u64, + node_ensure_ms: f64, + node_detach_count: u64, + node_detach_ms: f64, + node_attach_count: u64, + node_attach_ms: f64, + node_tombstone_count: u64, + node_tombstone_ms: f64, + node_last_change_count: u64, + node_last_change_ms: f64, + node_deleted_at_count: u64, + node_deleted_at_ms: f64, + payload_load_count: u64, + payload_load_ms: f64, + payload_set_count: u64, + payload_set_ms: f64, + index_record_count: u64, + index_record_ms: f64, +} + +impl PgAppendProfile { + fn new(batch_ops: usize, doc_dirty_before: bool, head_seq_before: u64) -> Self { + Self { + batch_ops, + doc_dirty_before, + head_seq_before, + ..Self::default() + } + } + + fn log(&self, doc_id: &str, inserted: usize) { + eprintln!( + "{}", + serde_json::json!({ + "kind": "treecrdt_postgres_append_profile", + "docId": doc_id, + "batchOps": self.batch_ops, + "insertedOps": inserted, + "docDirtyBefore": self.doc_dirty_before, + "headSeqBefore": self.head_seq_before, + "bulkInsertMs": self.bulk_insert_ms, + "bulkInsertedOps": self.bulk_inserted_ops, + "dedupeFilterMs": self.dedupe_filter_ms, + "materializeMs": self.materialize_ms, + "updateHeadMs": self.update_head_ms, + "fallbackMarkDirty": self.fallback_mark_dirty, + "nodeLoadCount": self.node_load_count, + "nodeLoadMs": self.node_load_ms, + "nodeEnsureCount": self.node_ensure_count, + "nodeEnsureMs": self.node_ensure_ms, + "nodeDetachCount": self.node_detach_count, + "nodeDetachMs": self.node_detach_ms, + "nodeAttachCount": self.node_attach_count, + "nodeAttachMs": self.node_attach_ms, + "nodeTombstoneCount": self.node_tombstone_count, + "nodeTombstoneMs": self.node_tombstone_ms, + "nodeLastChangeCount": self.node_last_change_count, + "nodeLastChangeMs": self.node_last_change_ms, + "nodeDeletedAtCount": self.node_deleted_at_count, + "nodeDeletedAtMs": self.node_deleted_at_ms, + "payloadLoadCount": self.payload_load_count, + "payloadLoadMs": self.payload_load_ms, + "payloadSetCount": self.payload_set_count, + "payloadSetMs": self.payload_set_ms, + "indexRecordCount": self.index_record_count, + "indexRecordMs": self.index_record_ms, + }) + ); + } +} + #[derive(Clone, Debug)] struct TreeMeta { dirty: bool, @@ -167,15 +263,25 @@ struct PgCtx { doc_id: String, client: Rc>, stmts: Rc>>, + append_profile: Option>>, } impl PgCtx { fn new(client: Rc>, doc_id: &str) -> Result { + Self::new_with_profile(client, doc_id, None) + } + + fn new_with_profile( + client: Rc>, + doc_id: &str, + append_profile: Option>>, + ) -> Result { ensure_doc_meta(&client, doc_id)?; Ok(Self { doc_id: doc_id.to_string(), client, stmts: Rc::new(RefCell::new(HashMap::new())), + append_profile, }) } @@ -220,6 +326,7 @@ impl PgNodeStore { } fn load_node_row(&self, node: NodeId) -> Result> { + let started_at = Instant::now(); let node_bytes = node_to_bytes(node); let mut c = self.ctx.client.borrow_mut(); let stmt = self.ctx.stmt( @@ -244,6 +351,12 @@ impl PgNodeStore { let tombstone: bool = row.get(2); let last_change: Option> = row.get(3); let deleted_at: Option> = row.get(4); + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_load_count += 1; + profile.node_load_ms += elapsed_ms; + } Ok(Some(CachedNodeRow { parent, order_key, @@ -297,6 +410,7 @@ impl treecrdt_core::NodeStore for PgNodeStore { return Ok(()); } + let started_at = Instant::now(); let node_bytes = node_to_bytes(node); let mut c = self.ctx.client.borrow_mut(); let stmt = self.ctx.stmt( @@ -320,6 +434,12 @@ impl treecrdt_core::NodeStore for PgNodeStore { }), ); } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_ensure_count += 1; + profile.node_ensure_ms += elapsed_ms; + } Ok(()) } @@ -369,6 +489,7 @@ impl treecrdt_core::NodeStore for PgNodeStore { return Ok(()); } self.ensure_node(node)?; + let started_at = Instant::now(); let node_bytes = node_to_bytes(node); let mut c = self.ctx.client.borrow_mut(); let stmt = self.ctx.stmt( @@ -384,6 +505,12 @@ impl treecrdt_core::NodeStore for PgNodeStore { row.parent = None; row.order_key = None; } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_detach_count += 1; + profile.node_detach_ms += elapsed_ms; + } Ok(()) } @@ -394,6 +521,7 @@ impl treecrdt_core::NodeStore for PgNodeStore { self.ensure_node(node)?; self.ensure_node(parent)?; + let started_at = Instant::now(); let node_bytes = node_to_bytes(node); let parent_bytes = node_to_bytes(parent); let mut c = self.ctx.client.borrow_mut(); @@ -419,6 +547,12 @@ impl treecrdt_core::NodeStore for PgNodeStore { row.parent = Some(parent); row.order_key = None; } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_attach_count += 1; + profile.node_attach_ms += elapsed_ms; + } return Ok(()); } @@ -443,6 +577,12 @@ impl treecrdt_core::NodeStore for PgNodeStore { row.parent = Some(parent); row.order_key = Some(order_key); } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_attach_count += 1; + profile.node_attach_ms += elapsed_ms; + } Ok(()) } @@ -455,6 +595,7 @@ impl treecrdt_core::NodeStore for PgNodeStore { fn set_tombstone(&mut self, node: NodeId, tombstone: bool) -> Result<()> { self.ensure_node(node)?; + let started_at = Instant::now(); let node_bytes = node_to_bytes(node); let mut c = self.ctx.client.borrow_mut(); let stmt = self.ctx.stmt( @@ -472,6 +613,12 @@ impl treecrdt_core::NodeStore for PgNodeStore { if let Some(Some(row)) = self.cache.borrow_mut().get_mut(&node) { row.tombstone = tombstone; } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_tombstone_count += 1; + profile.node_tombstone_ms += elapsed_ms; + } Ok(()) } @@ -488,6 +635,7 @@ impl treecrdt_core::NodeStore for PgNodeStore { fn merge_last_change(&mut self, node: NodeId, delta: &VersionVector) -> Result<()> { self.ensure_node(node)?; + let started_at = Instant::now(); let mut vv = self.last_change(node)?; vv.merge(delta); let bytes = vv_to_bytes(&vv)?; @@ -506,6 +654,12 @@ impl treecrdt_core::NodeStore for PgNodeStore { if let Some(Some(row)) = self.cache.borrow_mut().get_mut(&node) { row.last_change = Some(bytes); } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_last_change_count += 1; + profile.node_last_change_ms += elapsed_ms; + } Ok(()) } @@ -522,6 +676,7 @@ impl treecrdt_core::NodeStore for PgNodeStore { fn merge_deleted_at(&mut self, node: NodeId, delta: &VersionVector) -> Result<()> { self.ensure_node(node)?; + let started_at = Instant::now(); let mut vv = self.deleted_at(node)?.unwrap_or_else(VersionVector::new); vv.merge(delta); let bytes = vv_to_bytes(&vv)?; @@ -540,6 +695,12 @@ impl treecrdt_core::NodeStore for PgNodeStore { if let Some(Some(row)) = self.cache.borrow_mut().get_mut(&node) { row.deleted_at = Some(bytes); } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_deleted_at_count += 1; + profile.node_deleted_at_ms += elapsed_ms; + } Ok(()) } @@ -570,6 +731,7 @@ impl PgPayloadStore { } fn load_payload_row(&self, node: NodeId) -> Result> { + let started_at = Instant::now(); let node_bytes = node_to_bytes(node); let mut c = self.ctx.client.borrow_mut(); let stmt = self.ctx.stmt( @@ -588,6 +750,12 @@ impl PgPayloadStore { let last_lamport = row.get::<_, i64>(1).max(0) as Lamport; let last_replica: Vec = row.get(2); let last_counter = row.get::<_, i64>(3).max(0) as u64; + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.payload_load_count += 1; + profile.payload_load_ms += elapsed_ms; + } Ok(Some(CachedPayloadRow { payload, last_lamport, @@ -641,6 +809,7 @@ impl treecrdt_core::PayloadStore for PgPayloadStore { payload: Option>, writer: (Lamport, OperationId), ) -> Result<()> { + let started_at = Instant::now(); let node_bytes = node_to_bytes(node); let (lamport, id) = writer; let OperationId { replica, counter } = id; @@ -673,6 +842,12 @@ impl treecrdt_core::PayloadStore for PgPayloadStore { last_counter: counter, }), ); + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.payload_set_count += 1; + profile.payload_set_ms += elapsed_ms; + } Ok(()) } } @@ -702,6 +877,7 @@ impl treecrdt_core::ParentOpIndex for PgParentOpIndex { if parent == NodeId::TRASH { return Ok(()); } + let started_at = Instant::now(); let parent_bytes = node_to_bytes(parent); let op_ref = derive_op_ref_v0(&self.ctx.doc_id, op_id.replica.as_bytes(), op_id.counter); let op_ref_bytes = op_ref.as_slice(); @@ -722,6 +898,12 @@ impl treecrdt_core::ParentOpIndex for PgParentOpIndex { ], ) .map_err(storage_debug)?; + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.index_record_count += 1; + profile.index_record_ms += elapsed_ms; + } Ok(()) } } @@ -1145,8 +1327,13 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> payloads, )?; + let materialize_started_at = Instant::now(); let next = apply_incremental_ops(&mut crdt, &mut index, meta, ops)? .ok_or_else(|| Error::Storage("expected head op after materialization".into()))?; + if let Some(profile) = &ctx.append_profile { + profile.borrow_mut().materialize_ms += materialize_started_at.elapsed().as_secs_f64() * 1000.0; + } + let update_head_started_at = Instant::now(); update_tree_meta_head( &ctx.client, &ctx.doc_id, @@ -1155,6 +1342,9 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> next.counter, next.seq, )?; + if let Some(profile) = &ctx.append_profile { + profile.borrow_mut().update_head_ms += update_head_started_at.elapsed().as_secs_f64() * 1000.0; + } Ok(()) } @@ -1184,28 +1374,58 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation // Serialize per-doc writers across all server instances (incremental materialization updates // derived tables + head_seq and is not safe to run concurrently for the same doc_id). let meta = load_tree_meta_for_update(client, doc_id)?; - let ctx = PgCtx::new(client.clone(), doc_id)?; + let append_profile = append_profile_enabled().then(|| { + Rc::new(RefCell::new(PgAppendProfile::new( + ops.len(), + meta.dirty, + meta.head_seq(), + ))) + }); + let ctx = PgCtx::new_with_profile(client.clone(), doc_id, append_profile.clone())?; if meta.dirty { + let bulk_insert_started_at = Instant::now(); let inserted = { let mut c = client.borrow_mut(); bulk_insert_ops_in_tx(&ctx, &mut c, ops)? }; + if let Some(profile) = &append_profile { + let mut profile = profile.borrow_mut(); + profile.bulk_insert_ms += bulk_insert_started_at.elapsed().as_secs_f64() * 1000.0; + profile.bulk_inserted_ops += inserted.len(); + } if !inserted.is_empty() { set_tree_meta_dirty(client, doc_id, true)?; + if let Some(profile) = &append_profile { + profile.borrow_mut().fallback_mark_dirty = true; + } + } + let inserted_count = inserted.len().min(u64::MAX as usize) as u64; + if let Some(profile) = &append_profile { + profile.borrow().log(doc_id, inserted_count as usize); } - return Ok(inserted.len().min(u64::MAX as usize) as u64); + return Ok(inserted_count); } + let bulk_insert_started_at = Instant::now(); let inserted_op_refs = { let mut c = client.borrow_mut(); bulk_insert_ops_in_tx(&ctx, &mut c, ops)? }; + if let Some(profile) = &append_profile { + let mut profile = profile.borrow_mut(); + profile.bulk_insert_ms += bulk_insert_started_at.elapsed().as_secs_f64() * 1000.0; + profile.bulk_inserted_ops += inserted_op_refs.len(); + } if inserted_op_refs.is_empty() { + if let Some(profile) = &append_profile { + profile.borrow().log(doc_id, 0); + } return Ok(0); } + let dedupe_filter_started_at = Instant::now(); let inserted_ops: Vec = if inserted_op_refs.len() == ops.len() { ops.to_vec() } else { @@ -1219,8 +1439,14 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation .cloned() .collect() }; + if let Some(profile) = &append_profile { + profile.borrow_mut().dedupe_filter_ms += dedupe_filter_started_at.elapsed().as_secs_f64() * 1000.0; + } if inserted_ops.is_empty() { + if let Some(profile) = &append_profile { + profile.borrow().log(doc_id, 0); + } return Ok(0); } @@ -1230,9 +1456,16 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation || materialize_ops_in_order(ctx, &meta, inserted_ops), || { let _ = set_tree_meta_dirty(client, doc_id, true); + if let Some(profile) = &append_profile { + profile.borrow_mut().fallback_mark_dirty = true; + } }, ); + if let Some(profile) = &append_profile { + profile.borrow().log(doc_id, inserted); + } + Ok(inserted.min(u64::MAX as usize) as u64) } From 21a86b1a34f590976386eb39690b8987c022a0cf Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 20:22:45 +0100 Subject: [PATCH 25/58] perf(postgres): batch index and last-change writes --- packages/treecrdt-postgres-rs/src/store.rs | 174 ++++++++++++++++----- 1 file changed, 131 insertions(+), 43 deletions(-) diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index 301e35ba..1aac5dcb 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -312,16 +312,19 @@ struct CachedPayloadRow { last_counter: u64, } +#[derive(Clone)] struct PgNodeStore { ctx: PgCtx, - cache: RefCell>>, + cache: Rc>>>, + pending_last_change: Rc>>, } impl PgNodeStore { fn new(ctx: PgCtx) -> Self { Self { ctx, - cache: RefCell::new(HashMap::new()), + cache: Rc::new(RefCell::new(HashMap::new())), + pending_last_change: Rc::new(RefCell::new(HashSet::new())), } } @@ -374,11 +377,59 @@ impl PgNodeStore { self.cache.borrow_mut().insert(node, loaded.clone()); Ok(loaded) } + + fn flush_last_change(&self) -> Result<()> { + let dirty_nodes = { + let mut pending = self.pending_last_change.borrow_mut(); + std::mem::take(&mut *pending) + }; + if dirty_nodes.is_empty() { + return Ok(()); + } + + let started_at = Instant::now(); + let mut nodes: Vec> = Vec::with_capacity(dirty_nodes.len()); + let mut values: Vec> = Vec::with_capacity(dirty_nodes.len()); + { + let cache = self.cache.borrow(); + for node in dirty_nodes { + let Some(Some(row)) = cache.get(&node) else { + continue; + }; + let Some(last_change) = &row.last_change else { + continue; + }; + nodes.push(node_to_bytes(node).to_vec()); + values.push(last_change.clone()); + } + } + if nodes.is_empty() { + return Ok(()); + } + + let mut c = self.ctx.client.borrow_mut(); + let stmt = self.ctx.stmt( + &mut c, + "UPDATE treecrdt_nodes AS dst \ + SET last_change = src.last_change \ + FROM unnest($2::bytea[], $3::bytea[]) AS src(node, last_change) \ + WHERE dst.doc_id = $1 AND dst.node = src.node", + )?; + c.execute(&stmt, &[&self.ctx.doc_id, &nodes, &values]) + .map_err(storage_debug)?; + + if let Some(profile) = &self.ctx.append_profile { + profile.borrow_mut().node_last_change_ms += started_at.elapsed().as_secs_f64() * 1000.0; + } + + Ok(()) + } } impl treecrdt_core::NodeStore for PgNodeStore { fn reset(&mut self) -> Result<()> { self.cache.borrow_mut().clear(); + self.pending_last_change.borrow_mut().clear(); let mut c = self.ctx.client.borrow_mut(); let stmt = self.ctx.stmt(&mut c, "DELETE FROM treecrdt_nodes WHERE doc_id = $1")?; c.execute(&stmt, &[&self.ctx.doc_id]).map_err(storage_debug)?; @@ -640,20 +691,10 @@ impl treecrdt_core::NodeStore for PgNodeStore { vv.merge(delta); let bytes = vv_to_bytes(&vv)?; - let node_bytes = node_to_bytes(node); - let mut c = self.ctx.client.borrow_mut(); - let stmt = self.ctx.stmt( - &mut c, - "UPDATE treecrdt_nodes \ - SET last_change = $3 \ - WHERE doc_id = $1 AND node = $2", - )?; - c.execute(&stmt, &[&self.ctx.doc_id, &node_bytes.as_slice(), &bytes]) - .map_err(storage_debug)?; - if let Some(Some(row)) = self.cache.borrow_mut().get_mut(&node) { row.last_change = Some(bytes); } + self.pending_last_change.borrow_mut().insert(node); if let Some(profile) = &self.ctx.append_profile { let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; let mut profile = profile.borrow_mut(); @@ -854,16 +895,57 @@ impl treecrdt_core::PayloadStore for PgPayloadStore { struct PgParentOpIndex { ctx: PgCtx, + pending: Vec, } impl PgParentOpIndex { fn new(ctx: PgCtx) -> Self { - Self { ctx } + Self { + ctx, + pending: Vec::new(), + } + } + + fn flush(&mut self) -> Result<()> { + if self.pending.is_empty() { + return Ok(()); + } + + let started_at = Instant::now(); + let mut parents: Vec> = Vec::with_capacity(self.pending.len()); + let mut op_refs: Vec> = Vec::with_capacity(self.pending.len()); + let mut seqs: Vec = Vec::with_capacity(self.pending.len()); + for row in self.pending.drain(..) { + parents.push(row.parent); + op_refs.push(row.op_ref); + seqs.push(row.seq); + } + + let mut c = self.ctx.client.borrow_mut(); + let stmt = self.ctx.stmt( + &mut c, + "INSERT INTO treecrdt_oprefs_children(doc_id, parent, op_ref, seq) \ + SELECT $1, src.parent, src.op_ref, src.seq \ + FROM unnest($2::bytea[], $3::bytea[], $4::bigint[]) AS src(parent, op_ref, seq) \ + ON CONFLICT (doc_id, parent, op_ref) DO NOTHING", + )?; + c.execute(&stmt, &[&self.ctx.doc_id, &parents, &op_refs, &seqs]) + .map_err(storage_debug)?; + + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.index_record_count += parents.len() as u64; + profile.index_record_ms += elapsed_ms; + } + + Ok(()) } } impl treecrdt_core::ParentOpIndex for PgParentOpIndex { fn reset(&mut self) -> Result<()> { + self.pending.clear(); let mut c = self.ctx.client.borrow_mut(); let stmt = self.ctx.stmt( &mut c, @@ -877,37 +959,26 @@ impl treecrdt_core::ParentOpIndex for PgParentOpIndex { if parent == NodeId::TRASH { return Ok(()); } - let started_at = Instant::now(); - let parent_bytes = node_to_bytes(parent); - let op_ref = derive_op_ref_v0(&self.ctx.doc_id, op_id.replica.as_bytes(), op_id.counter); - let op_ref_bytes = op_ref.as_slice(); - - let mut c = self.ctx.client.borrow_mut(); - let stmt = self.ctx.stmt( - &mut c, - "INSERT INTO treecrdt_oprefs_children(doc_id, parent, op_ref, seq) VALUES ($1,$2,$3,$4) \ - ON CONFLICT (doc_id, parent, op_ref) DO NOTHING", - )?; - c.execute( - &stmt, - &[ - &self.ctx.doc_id, - &parent_bytes.as_slice(), - &op_ref_bytes, - &(seq as i64), - ], - ) - .map_err(storage_debug)?; - if let Some(profile) = &self.ctx.append_profile { - let elapsed_ms = started_at.elapsed().as_secs_f64() * 1000.0; - let mut profile = profile.borrow_mut(); - profile.index_record_count += 1; - profile.index_record_ms += elapsed_ms; + self.pending.push(PendingParentOpRefRow { + parent: node_to_bytes(parent).to_vec(), + op_ref: derive_op_ref_v0(&self.ctx.doc_id, op_id.replica.as_bytes(), op_id.counter).to_vec(), + seq: seq as i64, + }); + if self.pending.len() >= PARENT_OP_INDEX_FLUSH_SIZE { + self.flush()?; } Ok(()) } } +const PARENT_OP_INDEX_FLUSH_SIZE: usize = 4096; + +struct PendingParentOpRefRow { + parent: Vec, + op_ref: Vec, + seq: i64, +} + struct PgOpStorage { ctx: PgCtx, } @@ -1316,6 +1387,7 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> } let nodes = PgNodeStore::new(ctx.clone()); + let node_flush = nodes.clone(); let payloads = PgPayloadStore::new(ctx.clone()); let mut index = PgParentOpIndex::new(ctx.clone()); @@ -1330,6 +1402,8 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> let materialize_started_at = Instant::now(); let next = apply_incremental_ops(&mut crdt, &mut index, meta, ops)? .ok_or_else(|| Error::Storage("expected head op after materialization".into()))?; + node_flush.flush_last_change()?; + index.flush()?; if let Some(profile) = &ctx.append_profile { profile.borrow_mut().materialize_ms += materialize_started_at.elapsed().as_secs_f64() * 1000.0; } @@ -1522,6 +1596,7 @@ fn ensure_materialized_in_tx(client: &Rc>, doc_id: &str) -> Resu let ctx = PgCtx::new(client.clone(), doc_id)?; let storage = PgOpStorage::new(ctx.clone()); let mut nodes = PgNodeStore::new(ctx.clone()); + let node_flush = nodes.clone(); let mut payloads = PgPayloadStore::new(ctx.clone()); let mut index = PgParentOpIndex::new(ctx.clone()); @@ -1537,6 +1612,8 @@ fn ensure_materialized_in_tx(client: &Rc>, doc_id: &str) -> Resu payloads, )?; crdt.replay_from_storage_with_materialization(&mut index)?; + node_flush.flush_last_change()?; + index.flush()?; let seq = crdt.log_len().min(u64::MAX as usize) as u64; if let Some(last) = crdt.head_op() { @@ -1945,6 +2022,7 @@ type LocalCrdt = TreeCrdt seq = v, + Ok(v) => { + seq = v; + if session.nodes.flush_last_change().is_err() || op_index.flush().is_err() { + post_materialization_ok = false; + } + } Err(_) => post_materialization_ok = false, } From 4e325e0bb4b02d2529231b1d855cbc3ab41af957 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 22 Mar 2026 21:06:42 +0100 Subject: [PATCH 26/58] perf(postgres): preload nodes for payload batches --- packages/treecrdt-postgres-rs/src/store.rs | 134 +++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index 1aac5dcb..1c4ea5d8 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -378,6 +378,134 @@ impl PgNodeStore { Ok(loaded) } + fn preload_for_ops(&self, ops: &[Operation]) -> Result<()> { + let mut referenced: HashSet = HashSet::new(); + for op in ops { + let node = op.kind.node(); + if node != NodeId::TRASH { + referenced.insert(node); + } + match &op.kind { + OperationKind::Insert { parent, .. } => { + if *parent != NodeId::TRASH { + referenced.insert(*parent); + } + } + OperationKind::Move { new_parent, .. } => { + if *new_parent != NodeId::TRASH { + referenced.insert(*new_parent); + } + } + OperationKind::Delete { .. } + | OperationKind::Tombstone { .. } + | OperationKind::Payload { .. } => {} + } + } + + if referenced.is_empty() { + return Ok(()); + } + + let mut requested_nodes: Vec = referenced.iter().copied().collect(); + requested_nodes.sort(); + let requested_bytes: Vec> = + requested_nodes.iter().map(|node| node_to_bytes(*node).to_vec()).collect(); + + let load_started_at = Instant::now(); + let mut loaded_nodes: HashSet = HashSet::with_capacity(requested_nodes.len()); + { + let mut c = self.ctx.client.borrow_mut(); + let stmt = self.ctx.stmt( + &mut c, + "SELECT node, parent, order_key, tombstone, last_change, deleted_at \ + FROM treecrdt_nodes \ + WHERE doc_id = $1 \ + AND node IN (SELECT DISTINCT i.node FROM unnest($2::bytea[]) AS i(node))", + )?; + let rows = c + .query(&stmt, &[&self.ctx.doc_id, &requested_bytes]) + .map_err(storage_debug)?; + let mut cache = self.cache.borrow_mut(); + for row in rows { + let node_bytes: Vec = row.get(0); + let node = bytes_to_node(&node_bytes)?; + let parent_bytes: Option> = row.get(1); + let parent = match parent_bytes { + None => None, + Some(b) => Some(bytes_to_node(&b)?), + }; + let order_key: Option> = row.get(2); + let tombstone: bool = row.get(3); + let last_change: Option> = row.get(4); + let deleted_at: Option> = row.get(5); + loaded_nodes.insert(node); + cache.insert( + node, + Some(CachedNodeRow { + parent, + order_key, + tombstone, + last_change, + deleted_at, + }), + ); + } + } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = load_started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_load_count += loaded_nodes.len() as u64; + profile.node_load_ms += elapsed_ms; + } + + let missing: Vec = requested_nodes + .into_iter() + .filter(|node| !loaded_nodes.contains(node)) + .collect(); + if missing.is_empty() { + return Ok(()); + } + + let ensure_started_at = Instant::now(); + let missing_bytes: Vec> = + missing.iter().map(|node| node_to_bytes(*node).to_vec()).collect(); + { + let mut c = self.ctx.client.borrow_mut(); + let stmt = self.ctx.stmt( + &mut c, + "INSERT INTO treecrdt_nodes(doc_id, node) \ + SELECT $1, src.node \ + FROM unnest($2::bytea[]) AS src(node) \ + ON CONFLICT (doc_id, node) DO NOTHING", + )?; + c.execute(&stmt, &[&self.ctx.doc_id, &missing_bytes]) + .map_err(storage_debug)?; + } + { + let mut cache = self.cache.borrow_mut(); + for node in missing { + cache.insert( + node, + Some(CachedNodeRow { + parent: None, + order_key: None, + tombstone: false, + last_change: None, + deleted_at: None, + }), + ); + } + } + if let Some(profile) = &self.ctx.append_profile { + let elapsed_ms = ensure_started_at.elapsed().as_secs_f64() * 1000.0; + let mut profile = profile.borrow_mut(); + profile.node_ensure_count += missing_bytes.len() as u64; + profile.node_ensure_ms += elapsed_ms; + } + + Ok(()) + } + fn flush_last_change(&self) -> Result<()> { let dirty_nodes = { let mut pending = self.pending_last_change.borrow_mut(); @@ -1387,6 +1515,12 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> } let nodes = PgNodeStore::new(ctx.clone()); + if ops + .iter() + .any(|op| matches!(op.kind, OperationKind::Payload { .. })) + { + nodes.preload_for_ops(&ops)?; + } let node_flush = nodes.clone(); let payloads = PgPayloadStore::new(ctx.clone()); let mut index = PgParentOpIndex::new(ctx.clone()); From 839ce195cc2eca7efddd04d78e9a35d70f2253ea Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Mon, 23 Mar 2026 10:51:02 +0100 Subject: [PATCH 27/58] style(rust): format postgres changes --- packages/treecrdt-postgres-rs/src/store.rs | 28 +++++++++---------- .../tests/postgres_test.rs | 5 +++- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index 1c4ea5d8..a7d6de36 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -422,9 +422,8 @@ impl PgNodeStore { WHERE doc_id = $1 \ AND node IN (SELECT DISTINCT i.node FROM unnest($2::bytea[]) AS i(node))", )?; - let rows = c - .query(&stmt, &[&self.ctx.doc_id, &requested_bytes]) - .map_err(storage_debug)?; + let rows = + c.query(&stmt, &[&self.ctx.doc_id, &requested_bytes]).map_err(storage_debug)?; let mut cache = self.cache.borrow_mut(); for row in rows { let node_bytes: Vec = row.get(0); @@ -478,8 +477,7 @@ impl PgNodeStore { FROM unnest($2::bytea[]) AS src(node) \ ON CONFLICT (doc_id, node) DO NOTHING", )?; - c.execute(&stmt, &[&self.ctx.doc_id, &missing_bytes]) - .map_err(storage_debug)?; + c.execute(&stmt, &[&self.ctx.doc_id, &missing_bytes]).map_err(storage_debug)?; } { let mut cache = self.cache.borrow_mut(); @@ -543,8 +541,7 @@ impl PgNodeStore { FROM unnest($2::bytea[], $3::bytea[]) AS src(node, last_change) \ WHERE dst.doc_id = $1 AND dst.node = src.node", )?; - c.execute(&stmt, &[&self.ctx.doc_id, &nodes, &values]) - .map_err(storage_debug)?; + c.execute(&stmt, &[&self.ctx.doc_id, &nodes, &values]).map_err(storage_debug)?; if let Some(profile) = &self.ctx.append_profile { profile.borrow_mut().node_last_change_ms += started_at.elapsed().as_secs_f64() * 1000.0; @@ -1089,7 +1086,8 @@ impl treecrdt_core::ParentOpIndex for PgParentOpIndex { } self.pending.push(PendingParentOpRefRow { parent: node_to_bytes(parent).to_vec(), - op_ref: derive_op_ref_v0(&self.ctx.doc_id, op_id.replica.as_bytes(), op_id.counter).to_vec(), + op_ref: derive_op_ref_v0(&self.ctx.doc_id, op_id.replica.as_bytes(), op_id.counter) + .to_vec(), seq: seq as i64, }); if self.pending.len() >= PARENT_OP_INDEX_FLUSH_SIZE { @@ -1515,10 +1513,7 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> } let nodes = PgNodeStore::new(ctx.clone()); - if ops - .iter() - .any(|op| matches!(op.kind, OperationKind::Payload { .. })) - { + if ops.iter().any(|op| matches!(op.kind, OperationKind::Payload { .. })) { nodes.preload_for_ops(&ops)?; } let node_flush = nodes.clone(); @@ -1539,7 +1534,8 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> node_flush.flush_last_change()?; index.flush()?; if let Some(profile) = &ctx.append_profile { - profile.borrow_mut().materialize_ms += materialize_started_at.elapsed().as_secs_f64() * 1000.0; + profile.borrow_mut().materialize_ms += + materialize_started_at.elapsed().as_secs_f64() * 1000.0; } let update_head_started_at = Instant::now(); update_tree_meta_head( @@ -1551,7 +1547,8 @@ fn materialize_ops_in_order(ctx: PgCtx, meta: &TreeMeta, ops: Vec) -> next.seq, )?; if let Some(profile) = &ctx.append_profile { - profile.borrow_mut().update_head_ms += update_head_started_at.elapsed().as_secs_f64() * 1000.0; + profile.borrow_mut().update_head_ms += + update_head_started_at.elapsed().as_secs_f64() * 1000.0; } Ok(()) } @@ -1648,7 +1645,8 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation .collect() }; if let Some(profile) = &append_profile { - profile.borrow_mut().dedupe_filter_ms += dedupe_filter_started_at.elapsed().as_secs_f64() * 1000.0; + profile.borrow_mut().dedupe_filter_ms += + dedupe_filter_started_at.elapsed().as_secs_f64() * 1000.0; } if inserted_ops.is_empty() { diff --git a/packages/treecrdt-postgres-rs/tests/postgres_test.rs b/packages/treecrdt-postgres-rs/tests/postgres_test.rs index d40c81e2..b82b4b49 100644 --- a/packages/treecrdt-postgres-rs/tests/postgres_test.rs +++ b/packages/treecrdt-postgres-rs/tests/postgres_test.rs @@ -97,7 +97,10 @@ fn postgres_backend_large_append_rebuilds_materialized_views_on_demand() { let inserted = append_ops(&client, &doc_id, &ops).unwrap(); assert_eq!(inserted, op_count); - assert_eq!(list_op_refs_all(&client, &doc_id).unwrap().len(), op_count as usize); + assert_eq!( + list_op_refs_all(&client, &doc_id).unwrap().len(), + op_count as usize + ); assert_eq!(max_lamport(&client, &doc_id).unwrap(), op_count); let children = tree_children(&client, &doc_id, NodeId::ROOT).unwrap(); From e30562229cc0de91ed7d035ce748acfad357f12b Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Mon, 23 Mar 2026 10:55:41 +0100 Subject: [PATCH 28/58] fix(clippy): avoid op-ref allocation --- packages/treecrdt-postgres-rs/src/store.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index a7d6de36..ac5ef1a4 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -1639,7 +1639,8 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation .filter(|op| { let replica = op.meta.id.replica.as_bytes(); let counter = op.meta.id.counter; - inserted_op_refs.contains(&derive_op_ref_v0(&ctx.doc_id, replica, counter).to_vec()) + inserted_op_refs + .contains(derive_op_ref_v0(&ctx.doc_id, replica, counter).as_slice()) }) .cloned() .collect() From 16bd296ac94326574693cfeda72b83937061eb3e Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Mon, 23 Mar 2026 15:38:52 +0100 Subject: [PATCH 29/58] docs: clarify sync and materialization fast paths --- packages/sync/protocol/src/sync.ts | 9 +++++++++ packages/treecrdt-postgres-rs/src/store.rs | 3 +++ 2 files changed, 12 insertions(+) diff --git a/packages/sync/protocol/src/sync.ts b/packages/sync/protocol/src/sync.ts index 9a5c7074..875cea63 100644 --- a/packages/sync/protocol/src/sync.ts +++ b/packages/sync/protocol/src/sync.ts @@ -450,6 +450,11 @@ export class SyncPeer { filter: Filter, opts: SyncOnceOptions = {} ): Promise { + // syncOnce negotiates one of three wire modes for this filter: + // 1. the normal RIBLT reconcile path, + // 2. direct-send for small scoped reads, or + // 3. direct-send upload when the initiator is an empty receiver. + // The capability exchange below advertises support and lets the peer pick the cheaper mode. const filterId = randomId("f"); const round = 0; const maxLamport = await this.backend.maxLamport(); @@ -1442,6 +1447,8 @@ export class SyncPeer { transport: DuplexTransport>, batch: OpsBatch ): Promise { + // Apply batches sequentially per filter so a later done marker cannot overtake earlier ops. + // Upload completion only becomes observable after the full queue for that filter has finished. const previous = this.opsBatchQueues.get(batch.filterId) ?? Promise.resolve(); const current = previous .catch(() => { @@ -1518,6 +1525,8 @@ export class SyncPeer { await this.reprocessPendingOps(); const responderSession = this.responderSessions.get(batch.filterId); + // The empty done ack is only a completion signal. Send it after applyOps/reprocessPending + // finishes so "done" means every prior batch for this filter has been durably handled. if (!responderSession && this.responderAwaitingUploadAcks.has(batch.filterId) && batch.done) { this.responderAwaitingUploadAcks.delete(batch.filterId); await transport.send({ diff --git a/packages/treecrdt-postgres-rs/src/store.rs b/packages/treecrdt-postgres-rs/src/store.rs index ac5ef1a4..4257bddb 100644 --- a/packages/treecrdt-postgres-rs/src/store.rs +++ b/packages/treecrdt-postgres-rs/src/store.rs @@ -1588,6 +1588,9 @@ fn append_ops_in_tx(client: &Rc>, doc_id: &str, ops: &[Operation }); let ctx = PgCtx::new_with_profile(client.clone(), doc_id, append_profile.clone())?; + // treecrdt_ops is the source of truth. If the doc is already dirty, derived tables are stale, + // so large uploads only append to the op log and leave the doc dirty for a later rebuild-on-read. + // When the doc is clean we can incrementally materialize the inserted ops to keep reads warm. if meta.dirty { let bulk_insert_started_at = Instant::now(); let inserted = { From 76bf329a1ee44686228f68fec1402324da5c5303 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Tue, 24 Mar 2026 08:33:51 +0100 Subject: [PATCH 30/58] fix(sync): refresh replay capabilities on live pushes --- packages/sync/protocol/src/sync.ts | 43 ++++++++--- packages/sync/protocol/tests/smoke.test.ts | 84 ++++++++++++++++++++++ 2 files changed, 118 insertions(+), 9 deletions(-) diff --git a/packages/sync/protocol/src/sync.ts b/packages/sync/protocol/src/sync.ts index 875cea63..480ff104 100644 --- a/packages/sync/protocol/src/sync.ts +++ b/packages/sync/protocol/src/sync.ts @@ -286,6 +286,13 @@ function peerSelectedDirectSendEmptyReceiverFilter( ); } +function capabilitySetFingerprint(capabilities: readonly Capability[]): string { + return capabilities + .map((capability) => `${capability.name}\u0000${capability.value}`) + .sort() + .join("\u0001"); +} + export class SyncPeer { private readonly maxCodewords: number; private readonly maxOpsPerBatch: number; @@ -295,6 +302,7 @@ export class SyncPeer { private readonly auth?: SyncAuth; private readonly transportHasAuth = new WeakMap>, boolean>(); private readonly transportPeerCapabilities = new WeakMap>, Hello["capabilities"]>(); + private readonly transportLastSentHelloCaps = new WeakMap>, string>(); private readonly responderSessions = new Map>(); private readonly initiatorSessions = new Map>(); private readonly responderSubscriptions = new Map>(); @@ -372,6 +380,27 @@ export class SyncPeer { } } + private async refreshHelloCapabilities( + transport: DuplexTransport>, + opts: { force?: boolean } = {} + ): Promise { + if (!this.auth?.helloCapabilities) return; + + const [maxLamport, capabilities] = await Promise.all([ + this.backend.maxLamport(), + this.auth.helloCapabilities({ docId: this.backend.docId }), + ]); + const fingerprint = capabilitySetFingerprint(capabilities); + if (!opts.force && this.transportLastSentHelloCaps.get(transport) === fingerprint) return; + + await transport.send({ + v: 0, + docId: this.backend.docId, + payload: { case: "hello", value: { capabilities, filters: [], maxLamport } }, + }); + this.transportLastSentHelloCaps.set(transport, fingerprint); + } + private async pushSubscription(sub: ResponderSubscription): Promise { let opRefs: OpRef[]; try { @@ -389,6 +418,10 @@ export class SyncPeer { } if (newOpRefs.length === 0) return; + // Live subscriptions can outlive the capability snapshot from the initial handshake. + // Refresh it before the push so proof_ref verification can succeed on newly seen authors. + await this.refreshHelloCapabilities(sub.transport); + for (let start = 0; start < newOpRefs.length; start += this.maxOpsPerBatch) { const chunk = newOpRefs.slice(start, start + this.maxOpsPerBatch); let ops = await this.backend.getOpsByOpRefs(chunk); @@ -745,15 +778,7 @@ export class SyncPeer { // If the responder requires capability-gated filters/subscriptions, send an initial // Hello (no filters) so it can record our capabilities before Subscribe arrives. if (this.auth?.helloCapabilities) { - const [maxLamport, capabilities] = await Promise.all([ - this.backend.maxLamport(), - this.auth.helloCapabilities({ docId: this.backend.docId }), - ]); - await transport.send({ - v: 0, - docId: this.backend.docId, - payload: { case: "hello", value: { capabilities, filters: [], maxLamport } }, - }); + await this.refreshHelloCapabilities(transport, { force: true }); } await transport.send({ diff --git a/packages/sync/protocol/tests/smoke.test.ts b/packages/sync/protocol/tests/smoke.test.ts index 14cafd34..b1871e60 100644 --- a/packages/sync/protocol/tests/smoke.test.ts +++ b/packages/sync/protocol/tests/smoke.test.ts @@ -823,3 +823,87 @@ test("subscribe keeps peers converging (push deltas)", async () => { detach(); } }); + +test("subscribe refreshes replay capabilities before pushing newly authorized ops", async () => { + const docId = "doc-subscribe-capability-refresh"; + const root = "0".repeat(32); + const proofRef = new Uint8Array([0x12, 0x34, 0x56, 0x78]); + const replayCapValue = bytesToHex(proofRef); + + const a = new MemoryBackend(docId); + const b = new MemoryBackend(docId); + const [transportA, transportB, wire] = createLoggedTimedDuplex>(); + const knownReplayCaps = new Set(); + let serverHelloCaps: Array<{ name: string; value: string }> = []; + + const peerA = new SyncPeer(a, { + auth: { + helloCapabilities: async () => serverHelloCaps, + onHello: async () => serverHelloCaps, + signOps: async (ops) => ops.map(() => ({ sig: new Uint8Array([1]), proofRef })), + }, + maxOpsPerBatch: 1, + }); + const peerB = new SyncPeer(b, { + auth: { + helloCapabilities: async () => [{ name: "auth.capability", value: "client-token" }], + onHello: async (hello) => { + for (const cap of hello.capabilities) { + if (cap.name === "auth.capability.replay") knownReplayCaps.add(cap.value); + } + return []; + }, + verifyOps: async (_ops, auth) => { + if (!auth) throw new Error("expected auth on subscribed push"); + for (const entry of auth) { + if (!entry?.proofRef) throw new Error("expected proofRef on subscribed push"); + if (!knownReplayCaps.has(bytesToHex(entry.proofRef))) { + throw new Error("missing replay capability for pushed proofRef"); + } + } + }, + }, + maxOpsPerBatch: 1, + }); + + const detachA = peerA.attach(transportA); + const detachB = peerB.attach(transportB); + + try { + const sub = peerB.subscribe(transportB, { all: {} }, { immediate: false, intervalMs: 0 }); + try { + await waitUntil(() => (peerA as any).responderSubscriptions?.size === 1, { + message: "expected responder subscription to be registered", + }); + + serverHelloCaps = [{ name: "auth.capability.replay", value: replayCapValue }]; + await a.applyOps([ + makeOp(replicas.a, 1, 1, { + type: "insert", + parent: root, + node: nodeIdFromInt(11), + orderKey: orderKeyFromPosition(0), + }), + ]); + await peerA.notifyLocalUpdate(); + + await waitUntil(() => b.hasOp(replicaHex.a, 1), { + message: "expected subscriber to accept live op after capability refresh", + }); + expect(knownReplayCaps).toEqual(new Set([replayCapValue])); + + const serverCases = wire + .filter((entry) => entry.dir === "aToB") + .map((entry) => entry.msg.payload.case); + expect(serverCases).toContain("hello"); + expect(serverCases).toContain("opsBatch"); + expect(serverCases.indexOf("hello")).toBeLessThan(serverCases.lastIndexOf("opsBatch")); + } finally { + sub.stop(); + await sub.done; + } + } finally { + detachA(); + detachB(); + } +}); From 315c49e8062729fd4c51234521f5e9900ccc8122 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Tue, 24 Mar 2026 10:45:50 +0100 Subject: [PATCH 31/58] style(playground): stop rotating sync status icon --- examples/playground/src/playground/components/PeersPanel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/playground/src/playground/components/PeersPanel.tsx b/examples/playground/src/playground/components/PeersPanel.tsx index bd3f58c2..f0251a2b 100644 --- a/examples/playground/src/playground/components/PeersPanel.tsx +++ b/examples/playground/src/playground/components/PeersPanel.tsx @@ -35,7 +35,7 @@ function RemoteStatusIcon({ status }: { status: RemoteSyncStatus }) { return ; } if (status.state === "connecting") { - return ; + return ; } if (status.state === "disabled") { return ; From 01c36bfc7b25d812278e4c5a5f458291780555d2 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Tue, 24 Mar 2026 15:40:18 +0100 Subject: [PATCH 32/58] fix(playground): improve new-doc and remote sync flows --- examples/playground/src/App.tsx | 105 ++++- .../playground/components/ComposerPanel.tsx | 199 +++++++--- .../components/PlaygroundHeader.tsx | 10 +- .../src/playground/components/TreePanel.tsx | 2 +- .../src/playground/hooks/usePlaygroundSync.ts | 73 +++- examples/playground/src/playground/persist.ts | 8 +- package.json | 1 + packages/treecrdt-wa-sqlite/src/client.ts | 350 +++++++---------- scripts/bench-playground-live-write.mjs | 370 ++++++++++++++++++ 9 files changed, 814 insertions(+), 304 deletions(-) create mode 100644 scripts/bench-playground-live-write.mjs diff --git a/examples/playground/src/App.tsx b/examples/playground/src/App.tsx index bec96085..1366acd6 100644 --- a/examples/playground/src/App.tsx +++ b/examples/playground/src/App.tsx @@ -24,6 +24,7 @@ import { ensureOpfsKey, initialDocId, initialStorage, + makeDefaultDocId, makeNodeId, makeSessionKey, persistDocId, @@ -81,6 +82,13 @@ function initialSyncTransportMode(): SyncTransportMode { return "local"; } +type BulkAddProgress = { + total: number; + completed: number; + phase: "creating" | "applying"; + startedAtMs: number; +}; + export default function App() { const [client, setClient] = useState(null); const clientRef = useRef(null); @@ -104,6 +112,7 @@ export default function App() { overrides: new Set([ROOT_ID]), })); const [busy, setBusy] = useState(false); + const [bulkAddProgress, setBulkAddProgress] = useState(null); const [nodeCount, setNodeCount] = useState(1); const [fanout, setFanout] = useState(10); const [newNodeValue, setNewNodeValue] = useState(""); @@ -155,6 +164,8 @@ export default function App() { const counterRef = useRef(0); const lamportRef = useRef(0); + const initEpochRef = useRef(0); + const disposedRef = useRef(false); const opfsSupport = useMemo(detectOpfsSupport, []); const docPayloadKeyRef = useRef(null); const refreshDocPayloadKey = React.useCallback(async () => { @@ -723,13 +734,28 @@ export default function App() { persistDocId(docId); }, [docId]); + const closeClientSafely = React.useCallback(async (closingClient: TreecrdtClient | null) => { + if (!closingClient?.close) return; + try { + await closingClient.close(); + } catch { + // Client teardown is best-effort. HMR/remount/reset can race prior close calls. + } + }, []); + useEffect(() => { + disposedRef.current = false; return () => { - void clientRef.current?.close(); + disposedRef.current = true; + initEpochRef.current += 1; + const closingClient = clientRef.current; + clientRef.current = null; + void closeClientSafely(closingClient); }; - }, []); + }, [closeClientSafely]); - const initClient = async (storageMode: StorageMode, keyOverride?: string) => { + const initClient = async (storageMode: StorageMode, keyOverride?: string, docIdOverride?: string) => { + const initEpoch = ++initEpochRef.current; setStatus("booting"); setError(null); try { @@ -744,8 +770,12 @@ export default function App() { baseUrl, preferWorker: storageMode === "opfs", filename, - docId, + docId: docIdOverride ?? docId, }); + if (disposedRef.current || initEpoch !== initEpochRef.current) { + await closeClientSafely(c); + return; + } clientRef.current = c; setClient(c); setStorage(c.storage); @@ -760,7 +790,8 @@ export default function App() { } }; - const resetAndInit = async (target: StorageMode, opts: { resetKey?: boolean } = {}) => { + const resetAndInit = async (target: StorageMode, opts: { resetKey?: boolean; docId?: string } = {}) => { + setStatus("booting"); const nextKey = target === "opfs" ? opts.resetKey @@ -781,12 +812,15 @@ export default function App() { lamportRef.current = 0; setHeadLamport(0); setTotalNodes(null); - if (clientRef.current?.close) { - await clientRef.current.close(); - } + setParentChoice(ROOT_ID); + setNewNodeValue(""); + setBulkAddProgress(null); + setError(null); + const closingClient = clientRef.current; clientRef.current = null; setClient(null); - await initClient(target, nextKey); + await closeClientSafely(closingClient); + await initClient(target, nextKey, opts.docId); }; const refreshOps = async (nextClient?: TreecrdtClient, opts: { preserveParent?: boolean } = {}) => { @@ -833,7 +867,7 @@ export default function App() { counterRef.current = Math.max(counterRef.current, op.meta.id.counter); setHeadLamport(lamportRef.current); - notifyLocalUpdate(); + notifyLocalUpdate([op]); await refreshPayloadsForNodes(client, nodesAffectedByPayloadOps([op])); ingestOps([op], { assumeSorted: true }); scheduleRefreshParents(parentsAffectedByOps(stateBefore, [op])); @@ -854,7 +888,7 @@ export default function App() { const stateBefore = treeStateRef.current; const placement = after ? { type: "after" as const, after } : { type: "first" as const }; const op = await client.local.move(replica, nodeId, newParent, placement); - notifyLocalUpdate(); + notifyLocalUpdate([op]); ingestOps([op], { assumeSorted: true }); scheduleRefreshParents(parentsAffectedByOps(stateBefore, [op])); scheduleRefreshNodeCount(); @@ -875,6 +909,9 @@ export default function App() { const normalizedCount = Math.max(0, Math.min(MAX_COMPOSER_NODE_COUNT, Math.floor(count))); if (normalizedCount <= 0) return; setBusy(true); + const startedAtMs = Date.now(); + const progressStep = normalizedCount >= 1_000 ? 50 : normalizedCount >= 200 ? 20 : normalizedCount >= 50 ? 5 : 1; + setBulkAddProgress({ total: normalizedCount, completed: 0, phase: "creating", startedAtMs }); try { const stateBefore = treeStateRef.current; const ops: Operation[] = []; @@ -889,6 +926,12 @@ export default function App() { const payload = shouldSetValue ? textEncoder.encode(value) : null; const encryptedPayload = await encryptPayloadBytes(payload); ops.push(await client.local.insert(replica, parentId, nodeId, { type: "last" }, encryptedPayload)); + const completed = i + 1; + if (completed === normalizedCount || completed % progressStep === 0) { + setBulkAddProgress((prev) => + prev ? { ...prev, completed } : prev + ); + } } } else { const expanded = new Set(); @@ -933,16 +976,26 @@ export default function App() { setChildCount(targetParent, childCount + 1); queue.push(nodeId); + const completed = i + 1; + if (completed === normalizedCount || completed % progressStep === 0) { + setBulkAddProgress((prev) => + prev ? { ...prev, completed } : prev + ); + } } } + setBulkAddProgress((prev) => + prev ? { ...prev, completed: normalizedCount, phase: "applying" } : prev + ); + for (const op of ops) { lamportRef.current = Math.max(lamportRef.current, op.meta.lamport); counterRef.current = Math.max(counterRef.current, op.meta.id.counter); } setHeadLamport(lamportRef.current); - notifyLocalUpdate(); + notifyLocalUpdate(ops); await refreshPayloadsForNodes(client, nodesAffectedByPayloadOps(ops)); ingestOps(ops, { assumeSorted: true }); scheduleRefreshParents(parentsAffectedByOps(stateBefore, ops)); @@ -966,6 +1019,7 @@ export default function App() { console.error("Failed to add nodes", err); setError("Failed to add nodes (see console)"); } finally { + setBulkAddProgress(null); setBusy(false); } }; @@ -981,7 +1035,7 @@ export default function App() { const encryptedPayload = await encryptPayloadBytes(payload); const nodeId = makeNodeId(); const op = await client.local.insert(replica, parentId, nodeId, { type: "last" }, encryptedPayload); - notifyLocalUpdate(); + notifyLocalUpdate([op]); await refreshPayloadsForNodes(client, [op.kind.node]); ingestOps([op], { assumeSorted: true }); scheduleRefreshParents(parentsAffectedByOps(stateBefore, [op])); @@ -1053,6 +1107,29 @@ export default function App() { await resetAndInit(storage, { resetKey: true }); }; + const handleNewDoc = async () => { + if (typeof window !== "undefined") { + const url = new URL(window.location.href); + url.searchParams.set("doc", makeDefaultDocId()); + url.searchParams.delete("join"); + url.searchParams.delete("autosync"); + url.hash = ""; + window.history.replaceState({}, "", url); + const nextDocId = url.searchParams.get("doc")!; + setDocId(nextDocId); + setLiveChildrenParents(new Set()); + setLiveAllEnabled(false); + await resetAndInit(storage, { resetKey: true, docId: nextDocId }); + return; + } + + const nextDocId = makeDefaultDocId(); + setDocId(nextDocId); + setLiveChildrenParents(new Set()); + setLiveAllEnabled(false); + await resetAndInit(storage, { resetKey: true, docId: nextDocId }); + }; + const handleStorageToggle = (next: StorageMode) => { if (next === storage) { void handleReset(); @@ -1154,6 +1231,7 @@ export default function App() { ) } onSelectStorage={handleStorageToggle} + onNewDoc={handleNewDoc} onReset={handleReset} onExpandAll={expandAll} onCollapseAll={collapseAll} @@ -1178,6 +1256,7 @@ export default function App() { onAddNodes={handleAddNodes} ready={status === "ready"} busy={busy} + bulkAddProgress={bulkAddProgress} canWritePayload={canWritePayload} canWriteStructure={canWriteStructure} /> diff --git a/examples/playground/src/playground/components/ComposerPanel.tsx b/examples/playground/src/playground/components/ComposerPanel.tsx index 75ffa51c..e8705bed 100644 --- a/examples/playground/src/playground/components/ComposerPanel.tsx +++ b/examples/playground/src/playground/components/ComposerPanel.tsx @@ -3,6 +3,13 @@ import { MdExpandLess, MdExpandMore } from "react-icons/md"; import { ParentPicker } from "./ParentPicker"; +type BulkAddProgress = { + total: number; + completed: number; + phase: "creating" | "applying"; + startedAtMs: number; +}; + export function ComposerPanel({ composerOpen, setComposerOpen, @@ -19,6 +26,7 @@ export function ComposerPanel({ onAddNodes, ready, busy, + bulkAddProgress, canWritePayload, canWriteStructure, }: { @@ -37,11 +45,43 @@ export function ComposerPanel({ onAddNodes: (parentId: string, count: number, opts: { fanout: number }) => void | Promise; ready: boolean; busy: boolean; + bulkAddProgress: BulkAddProgress | null; canWritePayload: boolean; canWriteStructure: boolean; }) { const containerPadding = composerOpen ? "p-5" : "p-3"; const headerMargin = composerOpen ? "mb-3" : "mb-0"; + const [progressNowMs, setProgressNowMs] = React.useState(() => Date.now()); + + React.useEffect(() => { + if (!bulkAddProgress) return; + setProgressNowMs(Date.now()); + const timer = window.setInterval(() => setProgressNowMs(Date.now()), 200); + return () => window.clearInterval(timer); + }, [bulkAddProgress]); + + const elapsedMs = bulkAddProgress ? Math.max(0, progressNowMs - bulkAddProgress.startedAtMs) : 0; + const progressRatio = bulkAddProgress ? Math.min(1, bulkAddProgress.completed / Math.max(1, bulkAddProgress.total)) : 0; + const etaMs = + bulkAddProgress && + bulkAddProgress.phase === "creating" && + bulkAddProgress.completed > 0 && + bulkAddProgress.completed < bulkAddProgress.total + ? Math.round((elapsedMs / bulkAddProgress.completed) * (bulkAddProgress.total - bulkAddProgress.completed)) + : null; + + const formatDuration = (ms: number): string => { + const totalSeconds = Math.max(0, Math.round(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${String(seconds).padStart(2, "0")}s` : `${seconds}s`; + }; + + const submitLabel = bulkAddProgress + ? bulkAddProgress.phase === "creating" + ? `Loading ${Math.round(progressRatio * 100)}%` + : "Finishing..." + : `Add node${nodeCount > 1 ? "s" : ""}`; return (
{composerOpen ? ( -
{ - e.preventDefault(); - void onAddNodes(parentChoice, nodeCount, { fanout }); - }} - > - - - - - - + + + + + + + {bulkAddProgress ? ( +
+
+ + {bulkAddProgress.phase === "creating" + ? `Creating ${bulkAddProgress.completed} of ${bulkAddProgress.total} nodes` + : `Applying ${bulkAddProgress.total} generated nodes`} + + Elapsed {formatDuration(elapsedMs)} +
+
+
+
+
+ + {bulkAddProgress.phase === "creating" + ? "Preparing inserts locally before the tree view refreshes." + : "Finalizing local tree state and sync updates."} + + + {etaMs !== null ? `ETA ~${formatDuration(etaMs)}` : bulkAddProgress.phase === "applying" ? "ETA settling..." : ""} + +
+
+ ) : null} + ) : null}
); diff --git a/examples/playground/src/playground/components/PlaygroundHeader.tsx b/examples/playground/src/playground/components/PlaygroundHeader.tsx index 18cc01af..ed460fbe 100644 --- a/examples/playground/src/playground/components/PlaygroundHeader.tsx +++ b/examples/playground/src/playground/components/PlaygroundHeader.tsx @@ -13,6 +13,7 @@ export function PlaygroundHeader({ selfPeerIdShort, onCopyPubkey, onSelectStorage, + onNewDoc, onReset, onExpandAll, onCollapseAll, @@ -27,6 +28,7 @@ export function PlaygroundHeader({ selfPeerIdShort: string | null; onCopyPubkey: () => void; onSelectStorage: (next: StorageMode) => void; + onNewDoc: () => void; onReset: () => void; onExpandAll: () => void; onCollapseAll: () => void; @@ -127,6 +129,13 @@ export function PlaygroundHeader({
+