Remove distributed benchmark harness (#4967)

# Description of Changes

Never used

# API and ABI breaking changes

N/A

# Expected complexity level and risk

0

# Testing

N/A

Author: joshua-spacetime

templates/keynote-2/.env.example

@@ -5,7 +5,6 @@
 #SKIP_SQLITE=1         # 1 = don't init SQLite in prep
 #SKIP_SUPABASE=1       # 1 = don't init Supabase in prep
 #SKIP_CONVEX=1          # 1 = don't init Convex in prep
-SPACETIME_METRICS_ENDPOINT=0
 
 # ===== PostgreSQL =====
 POSTGRES_USER=postgres

templates/keynote-2/DEVELOP.md

@@ -75,7 +75,8 @@ Copy `.env.example` to `.env` and adjust.
 - `SKIP_SQLITE` – `1` = don't init SQLite in prep
 - `SKIP_SUPABASE` – `1` = don't init Supabase in prep
 - `SKIP_CONVEX` – `1` = don't init Convex in prep
-- `SPACETIME_METRICS_ENDPOINT` – `1` = read committed transfer counts from the derived SpacetimeDB metrics endpoint; otherwise only local counters are used
+
+Throughput is counted from successful operations that the benchmark client observes completing inside the configured test window for every connector, including SpacetimeDB.
 
 **PostgreSQL / CockroachDB:**
 
@@ -166,7 +167,7 @@ cd ..
 
 ## Commands & Examples
 
-### 1. Run a test
+### Run a test
 
 ```bash
 pnpm run bench [test-name] [--seconds N] [--concurrency N] [--alpha A] [--connectors list] [--stdb-compression none|gzip]
@@ -194,190 +195,6 @@ pnpm run bench test-1 --connectors spacetimedb --stdb-compression gzip
 pnpm run bench test-1 --connectors spacetimedb,sqlite_rpc
 ```
 
-### 2. Run the distributed TypeScript SpacetimeDB benchmark
-
-Use this mode when you want to spread explicit TypeScript client connections across multiple machines. The existing `pnpm run bench` flow is still the single-process benchmark; the distributed flow is a separate coordinator + generator setup.
-
-The commands below are written so they run unchanged on a single machine. For a true multi-machine run, replace `127.0.0.1` with the actual coordinator and server hostnames or IP addresses reachable from each generator machine.
-
-#### Machine roles
-
-- **Server machine**: runs SpacetimeDB and hosts the benchmarked module.
-- **Coordinator machine**: runs `bench-dist-coordinator` and `bench-dist-control`. It may also run one or more generators if you want.
-- **Generator machines**: run `bench-dist-generator`. You can run multiple generator processes on the same machine as long as each one has a unique `--id`.
-
-#### Distributed setup
-
-All coordinator and generator machines should use the same `templates/keynote-2` checkout and have dependencies installed:
-
-```bash
-cd templates/keynote-2
-pnpm install
-cp .env.example .env
-```
-
-Generate TypeScript bindings in that checkout on each machine that will run the coordinator or a generator:
-
-```bash
-spacetime generate --lang typescript --out-dir module_bindings --module-path ./spacetimedb
-```
-
-#### Step 1: Start the server
-
-On the **server machine**:
-
-```bash
-spacetime start
-```
-
-#### Step 2: Publish the module and seed the accounts
-
-On any machine with the repo checkout and CLI access to the server, publish the module and seed the database once before starting the distributed run:
-
-```bash
-cd templates/keynote-2
-
-export STDB_URL=ws://127.0.0.1:3000
-export STDB_MODULE=test-1
-export STDB_MODULE_PATH=./spacetimedb
-
-spacetime publish -c -y --server local --module-path "$STDB_MODULE_PATH" "$STDB_MODULE"
-spacetime call --server local "$STDB_MODULE" seed 100000 10000000
-```
-
-If you are using a named server instead of `local`, replace `--server local` with the correct server name.
-
-#### Step 3: Start the coordinator
-
-On the **coordinator machine**:
-
-```bash
-cd templates/keynote-2
-
-pnpm run bench-dist-coordinator -- \
-  --test test-1 \
-  --connector spacetimedb \
-  --window-seconds 30 \
-  --verify 1 \
-  --stdb-url ws://127.0.0.1:3000 \
-  --stdb-module test-1 \
-  --stdb-compression none \
-  --bind 127.0.0.1 \
-  --port 8080
-```
-
-Notes:
-
-- Before measurement begins, the coordinator waits for every participating generator to start its epoch and acknowledge that it is running.
-- `--window-seconds` is the measured interval.
-- `--verify 1` preserves the existing benchmark semantics by running one verification pass centrally after the epoch completes.
-- If a generator never acknowledges start, the coordinator fails the epoch after `--start-ack-timeout-seconds` seconds. The default is `60`.
-- The coordinator derives the HTTP metrics endpoint from `--stdb-url` by switching to `http://` or `https://` and appending `/v1/metrics`.
-- For a real multi-machine run, change `--bind 127.0.0.1` to `--bind 0.0.0.0` so remote generators can reach the coordinator.
-- For a real multi-machine run, set `--stdb-url` to the server machine's reachable address.
-
-#### Step 4: Start generators on one or more client machines
-
-On **generator machine 1**:
-
-```bash
-cd templates/keynote-2
-
-pnpm run bench-dist-generator -- \
-  --id gen-a \
-  --coordinator-url http://127.0.0.1:8080 \
-  --test test-1 \
-  --connector spacetimedb \
-  --concurrency 2500 \
-  --accounts 100000 \
-  --alpha 1.5 \
-  --open-parallelism 128 \
-  --control-retries 3 \
-  --stdb-url ws://127.0.0.1:3000 \
-  --stdb-module test-1 \
-  --stdb-compression none
-```
-
-On **generator machine 2**:
-
-```bash
-cd templates/keynote-2
-
-pnpm run bench-dist-generator -- \
-  --id gen-b \
-  --coordinator-url http://127.0.0.1:8080 \
-  --test test-1 \
-  --connector spacetimedb \
-  --concurrency 2500 \
-  --accounts 100000 \
-  --alpha 1.5 \
-  --open-parallelism 128 \
-  --control-retries 3 \
-  --stdb-url ws://127.0.0.1:3000 \
-  --stdb-module test-1 \
-  --stdb-compression none
-```
-
-Repeat that on as many generator machines as needed, adjusting `--id` and `--concurrency` for each process.
-For a real multi-machine run, replace `127.0.0.1` with the coordinator host in `--coordinator-url` and the SpacetimeDB server host in `--stdb-url`.
-
-`--open-parallelism` controls connection ramp-up only. It deliberately avoids a connection storm by opening connections in bounded parallel batches.
-`--control-retries` sets the retry cap for `register`, `ready`, `/state`, and `/stopped`. The default is `3`.
-
-#### Step 5: Confirm generators are ready
-
-On the **coordinator machine**:
-
-```bash
-cd templates/keynote-2
-
-pnpm run bench-dist-control -- status --coordinator-url http://127.0.0.1:8080
-```
-
-Wait until each generator shows `state=ready` and `opened=N/N`.
-
-#### Step 6: Start an epoch
-
-On the **coordinator machine**:
-
-```bash
-cd templates/keynote-2
-
-pnpm run bench-dist-control -- start-epoch --coordinator-url http://127.0.0.1:8080 --label run-1
-```
-
-`start-epoch` waits for the epoch to finish, then prints the final result.
-
-#### Step 7: Check results
-
-Each completed epoch writes one JSON result file on the coordinator machine under:
-
-```text
-templates/keynote-2/runs/distributed/
-```
-
-The result contains:
-
-- participating generator IDs
-- total participating connections
-- committed transaction delta from the server metrics endpoint
-- measured window duration
-- computed TPS
-- verification result
-
-#### Operational notes
-
-- Start the coordinator before the generators.
-- Generators begin submitting requests when the coordinator enters `starting`.
-- Throughput is measured only from the committed transaction counter delta recorded after all participating generators have acknowledged start, so startup traffic is excluded.
-- For this distributed TypeScript mode, each connection runs closed-loop with one request at a time. There is no pipelining in this flow.
-- Late generators are allowed to register and become ready while an epoch is already running, but they only participate in the next epoch.
-- The coordinator does not use heartbeats. It includes generators that most recently reported `ready`.
-- If a participating generator dies and never sends `/stopped`, the epoch result is written with an `error`, and that generator remains `running` in coordinator status until you restart it and let it register again.
-- You can run multiple generator processes on the same machine if you want to test the harness locally. Just make sure each process uses a unique `--id`.
-
----
-
 ## CLI Arguments
 
 From `src/cli.ts`:

templates/keynote-2/README.md

@@ -35,8 +35,6 @@ All tests use 50 concurrent connections with a transfer workload (read-modify-wr
 
 **Key Finding:** SpacetimeDB achieves **~14x higher throughput** than the next best option (SQLite RPC) and maintains nearly identical performance under high contention (only ~4% drop), while traditional databases suffer significant degradation (CockroachDB drops 96%).
 
-> **Note:** SpacetimeDB runs on ARM architectures (including Apple M-series Macs), but has not yet been optimized for them.
-
 ### Contention Impact
 
 ![Contention Chart](./contention-chart.png)
@@ -69,6 +67,8 @@ Client → Integrated Platform (compute + storage colocated)
 
 This ensures we're measuring real-world application performance, not raw database throughput.
 
+Throughput is counted from successful operations that the benchmark client observes completing inside the configured test window for every system.
+
 ### The Transaction
 
 Each transaction performs a **fund transfer** between two accounts:
@@ -175,7 +175,6 @@ PlanetScale results (~477 TPS) demonstrate the **significant impact of cloud dat
 ## Running the Benchmarks
 
 See [DEVELOP.md](./DEVELOP.md) for prerequisites, configuration, and full CLI reference.
-The distributed TypeScript SpacetimeDB workflow is documented there as `Run the distributed TypeScript SpacetimeDB benchmark`.
 
 ## Output
 

templates/keynote-2/docker-compose-linux-raid-crdb.yml

@@ -174,7 +174,6 @@
       STDB_URL: ${STDB_URL}
       STDB_MODULE: ${STDB_MODULE}
       STDB_MODULE_PATH: ${STDB_MODULE_PATH}
-      STDB_METRICS_URL: ${STDB_METRICS_URL}
       STDB_CONFIRMED_READS: ${STDB_CONFIRMED_READS}
       BUN_URL: ${BUN_URL}
       SQLITE_FILE: /data/accounts.sqlite
@@ -189,7 +188,6 @@
       PLANETSCALE_RPC_URL: ${PLANETSCALE_RPC_URL}
       SEED_ACCOUNTS: ${SEED_ACCOUNTS}
       SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE}
-      USE_SPACETIME_METRICS_ENDPOINT: ${USE_SPACETIME_METRICS_ENDPOINT}
       CONVEX_USE_SHARDED_COUNTER: ${CONVEX_USE_SHARDED_COUNTER}
       VERIFY: ${VERIFY}
     volumes:

templates/keynote-2/docker-compose-linux-raid.yml

@@ -292,7 +292,6 @@
       STDB_URL: ${STDB_URL}
       STDB_MODULE: ${STDB_MODULE}
       STDB_MODULE_PATH: ${STDB_MODULE_PATH}
-      STDB_METRICS_URL: ${STDB_METRICS_URL}
       STDB_CONFIRMED_READS: ${STDB_CONFIRMED_READS}
       BUN_URL: ${BUN_URL}
       SQLITE_FILE: /data/accounts.sqlite
@@ -307,7 +306,6 @@
       PLANETSCALE_RPC_URL: ${PLANETSCALE_RPC_URL}
       SEED_ACCOUNTS: ${SEED_ACCOUNTS}
       SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE}
-      USE_SPACETIME_METRICS_ENDPOINT: ${USE_SPACETIME_METRICS_ENDPOINT}
       CONVEX_USE_SHARDED_COUNTER: ${CONVEX_USE_SHARDED_COUNTER}
       VERIFY: ${VERIFY}
       BENCH_PIPELINED: ${BENCH_PIPELINED}

templates/keynote-2/docker-compose.yml

@@ -257,7 +257,6 @@
       STDB_URL: ${STDB_URL}
       STDB_MODULE: ${STDB_MODULE}
       STDB_MODULE_PATH: ${STDB_MODULE_PATH}
-      STDB_METRICS_URL: ${STDB_METRICS_URL}
       STDB_CONFIRMED_READS: ${STDB_CONFIRMED_READS}
       BUN_URL: ${BUN_URL}
       SQLITE_FILE: ${SQLITE_FILE}
@@ -271,7 +270,6 @@
       PLANETSCALE_RPC_URL: ${PLANETSCALE_RPC_URL}
       SEED_ACCOUNTS: ${SEED_ACCOUNTS}
       SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE}
-      USE_SPACETIME_METRICS_ENDPOINT: ${USE_SPACETIME_METRICS_ENDPOINT}
       CONVEX_USE_SHARDED_COUNTER: ${CONVEX_USE_SHARDED_COUNTER}
       VERIFY: ${VERIFY}
     volumes:

templates/keynote-2/package.json

@@ -11,10 +11,7 @@
     "prep": "tsx src/init/init-all.ts",
     "down": "docker compose down || exit 0",
     "test-1": "tsx src/cli.ts test-1",
-    "bench": "tsx src/cli.ts",
-    "bench-dist-coordinator": "tsx src/distributed/coordinator.ts",
-    "bench-dist-generator": "tsx src/distributed/generator.ts",
-    "bench-dist-control": "tsx src/distributed/control.ts"
+    "bench": "tsx src/cli.ts"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",

templates/keynote-2/src/cli.ts

@@ -42,7 +42,6 @@ class BenchmarkTester {
     let totals = {
       tps: 0,
       samples: 0,
-      committed_txns: 0,
       p50_ms: 0,
       p95_ms: 0,
       p99_ms: 0,
@@ -54,7 +53,6 @@ class BenchmarkTester {
       const result = await runOne({ ...this.config, concurrency, alpha });
       totals.tps += result.tps;
       totals.samples += result.samples;
-      totals.committed_txns += result.committed_txns ?? 0;
       totals.p50_ms += result.p50_ms;
       totals.p95_ms += result.p95_ms;
       totals.p99_ms += result.p99_ms;
@@ -67,7 +65,6 @@ class BenchmarkTester {
     const avg = {
       tps: totals.tps / runs,
       samples: totals.samples / runs,
-      committed_txns: totals.committed_txns / runs,
       p50_ms: totals.p50_ms / runs,
       p95_ms: totals.p95_ms / runs,
       p99_ms: totals.p99_ms / runs,

templates/keynote-2/src/config.ts

@@ -45,7 +45,6 @@ export interface SharedRuntimeConfig {
   stdbCompression: StdbCompression;
   stdbConfirmedReads: boolean;
   useDocker: boolean;
-  useSpacetimeMetricsEndpoint: boolean;
   poolMax: number;
   bunUrl: string;
   convexUrl: string;
@@ -108,9 +107,7 @@ export type RunnerRuntimeConfig = Pick<
   | 'minOpTimeoutMs'
   | 'opTimeoutMs'
   | 'precomputedTransferPairs'
-  | 'stdbUrl'
   | 'tailSlackMs'
-  | 'useSpacetimeMetricsEndpoint'
   | 'verifyTransactions'
 >;
 
@@ -235,11 +232,6 @@ export function getSharedRuntimeDefaults(
     ),
     stdbConfirmedReads: readBooleanEnv('STDB_CONFIRMED_READS', true, env),
     useDocker: readBooleanEnv('USE_DOCKER', false, env),
-    useSpacetimeMetricsEndpoint: readBooleanEnv(
-      'SPACETIME_METRICS_ENDPOINT',
-      true,
-      env,
-    ),
     poolMax: readNumberEnv('MAX_POOL', 1000, env),
     bunUrl: readStringEnv('BUN_URL', 'http://127.0.0.1:4000', env),
     convexUrl: readStringEnv('CONVEX_URL', 'http://127.0.0.1:3210', env),

templates/keynote-2/src/connectors/spacetimedb.ts

@@ -78,7 +78,7 @@ export function spacetimedb(config: SpacetimeConnectorConfig): ReducerConnector
 
   return {
     name: 'spacetimedb',
-    maxInflightPerWorker: 512,
+    maxInflightPerWorker: 128,
 
     async open() {
       try {