Merge pull request #424 from cipherstash/dan/bench-package

perf(bench): add @cipherstash/bench for index-engagement validation

Author: coderdan

packages/bench/.gitignore

@@ -0,0 +1 @@
+results/

packages/bench/README.md

@@ -0,0 +1,49 @@
+# @cipherstash/bench
+
+Performance / index-engagement benchmarks for stack integrations.
+
+This package validates that each integration emits SQL that engages the canonical
+EQL functional indexes (`eql_v2.hmac_256`, `eql_v2.bloom_filter`, `eql_v2.ste_vec`)
+on a Supabase-shaped install (no operator classes). It runs in two layers:
+
+1. **EXPLAIN-shape tests** (`__tests__/`) — vitest tests that assert on
+   `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` output. Pass/fail. Cheap.
+2. **Wall-clock benches** (`__benches__/`) — vitest `--bench` (tinybench)
+   measuring median / p95 latency. On-demand; emits JSON to `results/`.
+
+## Prerequisites
+
+- Local Postgres + EQL via the repo-root `local/docker-compose.yml`:
+  ```bash
+  cd ../../local && docker compose up -d
+  ```
+- A CipherStash profile signed in (`stash login`). Auth is read from the
+  CipherStash profile; no environment variables required.
+- `DATABASE_URL` only needs to be set if you want to override the default
+  (`postgres://cipherstash:password@localhost:5432/cipherstash`).
+
+## Run
+
+The bench package's tests are **developer-run only** — they're not invoked by
+the repo's CI `test` step (the scripts are deliberately named `test:local` /
+`bench:local` so turbo's default `test` task skips this package).
+
+```bash
+# Credential-free smoke (verifies schema + EXPLAIN harness):
+pnpm test:local -- db-only
+
+# Full suite (requires CipherStash auth via `stash login`, seeds 10k rows on first run):
+pnpm db:setup                   # apply schema + seed BENCH_ROWS rows (default 10k)
+pnpm test:local                 # EXPLAIN-shape assertions for #421 / #422
+pnpm bench:local                # timing benches (slow)
+pnpm db:reset                   # drop schema (keeps EQL install)
+```
+
+`__tests__/db-only.test.ts` only touches Postgres + the EQL install and is the
+recommended starter — it's enough to verify the harness locally before wiring
+auth. The other tests under `__tests__/` and the benches under `__benches__/`
+use `@cipherstash/stack`'s `Encryption` client for real encryption.
+
+## Why this exists
+
+See cipherstash/stack issues #420, #421, #422.

packages/bench/__benches__/drizzle/operators.bench.ts

@@ -0,0 +1,69 @@
+import { createEncryptionOperators } from '@cipherstash/stack/drizzle'
+import type { SQL } from 'drizzle-orm'
+import { afterAll, beforeAll, bench, describe } from 'vitest'
+import {
+  type BenchHandle,
+  benchTable,
+  buildBench,
+  teardownBench,
+} from '../../src/drizzle/setup.js'
+import { applySchema } from '../../src/harness/db.js'
+import { seed } from '../../src/harness/seed.js'
+
+let handle: BenchHandle
+let ops: ReturnType<typeof createEncryptionOperators>
+
+beforeAll(async () => {
+  handle = await buildBench()
+  await applySchema(handle.pgClient)
+  await seed(handle)
+  ops = createEncryptionOperators(handle.encryptionClient)
+})
+
+afterAll(async () => {
+  if (handle) await teardownBench(handle)
+})
+
+/**
+ * Encryption cost is paid inside each iteration too — folding it into the
+ * timed loop reflects what customer code actually does, and the index
+ * engagement signal still dominates the differential between operators.
+ */
+describe('drizzle', () => {
+  bench('eq (string match)', async () => {
+    const where = (await ops.eq(benchTable.encText, 'value-0000042')) as SQL
+    await handle.db.select().from(benchTable).where(where)
+  })
+
+  bench('inArray (3 string matches)', async () => {
+    const where = await ops.inArray(benchTable.encText, [
+      'value-0000042',
+      'value-0000123',
+      'value-0000999',
+    ])
+    await handle.db.select().from(benchTable).where(where)
+  })
+
+  bench('like (prefix)', async () => {
+    const where = (await ops.like(benchTable.encText, '%value-00000%')) as SQL
+    await handle.db.select().from(benchTable).where(where)
+  })
+
+  bench('gt (int)', async () => {
+    const where = (await ops.gt(benchTable.encInt, 9990)) as SQL
+    await handle.db.select().from(benchTable).where(where)
+  })
+
+  bench('between (int)', async () => {
+    const where = (await ops.between(benchTable.encInt, 4000, 4100)) as SQL
+    await handle.db.select().from(benchTable).where(where)
+  })
+
+  bench('orderBy desc + limit 10', async () => {
+    await handle.db
+      .select()
+      .from(benchTable)
+      .orderBy(ops.desc(benchTable.encInt))
+      .limit(10)
+  })
+})

packages/bench/__tests__/db-only.test.ts

@@ -0,0 +1,56 @@
+/**
+ * DB-only smoke tests — exercise the schema/mode/EXPLAIN path against the
+ * existing local-postgres container without requiring CipherStash credentials.
+ * The seed/encryption path is covered separately by `harness.test.ts`, which
+ * does require credentials.
+ */
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { applySchema, connect, countBenchRows } from '../src/harness/db.js'
+import { explain, hasNodeType, summarize } from '../src/harness/explain.js'
+import type pg from 'pg'
+
+let client: pg.Client
+
+beforeAll(async () => {
+  client = await connect()
+  await applySchema(client)
+})
+
+afterAll(async () => {
+  if (client) await client.end()
+})
+
+describe('db-only harness', () => {
+  it('schema applied (bench table exists, count is 0)', async () => {
+    const rows = await countBenchRows(client)
+    expect(rows).toBe(0)
+  })
+
+  it('EXPLAIN parses a trivial plan', async () => {
+    const plan = await explain(client, 'SELECT id FROM bench LIMIT 1', [], {
+      analyze: false,
+    })
+    expect(plan['Node Type']).toBeTruthy()
+    expect(typeof summarize(plan)).toBe('string')
+  })
+
+  it('functional indexes exist after schema apply', async () => {
+    const res = await client.query<{ indexname: string }>(
+      `SELECT indexname FROM pg_indexes WHERE tablename = 'bench' ORDER BY indexname`,
+    )
+    const names = res.rows.map((r) => r.indexname)
+    expect(names).toContain('bench_text_hmac_idx')
+    expect(names).toContain('bench_text_bloom_idx')
+    expect(names).toContain('bench_jsonb_stevec_idx')
+  })
+
+  it('plan walker traverses nested Plans nodes', async () => {
+    const plan = await explain(
+      client,
+      'SELECT b1.id FROM bench b1 JOIN bench b2 ON b1.id = b2.id LIMIT 1',
+      [],
+      { analyze: false },
+    )
+    expect(hasNodeType(plan, 'Limit')).toBe(true)
+  })
+})