Merge remote-tracking branch 'origin/feat/eql-v3-text-search-schema' into feat/eql-v3-json

# Conflicts:
#	packages/stack/src/eql/v3/drizzle/operators.ts

Author: coderdan

.changeset/eql-v3-adapter-type-robustness.md

@@ -0,0 +1,38 @@
+---
+'@cipherstash/stack': minor
+---
+
+Restore the EQL v3 envelope and `Result` types the adapters were erasing.
+
+Both v3 adapters typed their operand-encryption paths as `unknown` and dropped
+the `Result` wrapper, so the query-type encoding and the failure channel were
+invisible to the type system:
+
+- `eql/v3/drizzle/operators.ts` typed the client's `encrypt`/`bulkEncrypt` as
+  returning `unknown`, collapsed the operation's `Result` to
+  `{ data?: unknown; failure?: { message } }`, and cast the bulk response to
+  `Array<{ data: unknown }>`.
+- `supabase/query-builder-v3.ts` returned `Promise<unknown[]>` from
+  `encryptCollectedTerms`, `bulkEncryptGroup` and `encryptGroupPerTerm`, and the
+  base `query-builder.ts` did the same.
+
+These now carry the SDK's real types — `Encrypted` (the storage envelope union,
+which includes every v3 per-domain payload), `BulkEncryptedData`, and
+`EncryptedQueryResult` — threaded through a properly-typed operation surface that
+resolves `Result<T, EncryptionError>`. The Supabase divergence the erasure hid is
+now explicit: the v2 path yields `encryptQuery` composite literals and the v3
+path yields `JSON.stringify`'d envelope strings, and both are `EncryptedQueryResult`.
+
+Bumped `minor`, not `patch`: `createEncryptionOperatorsV3` is a public export
+(`@cipherstash/stack/eql/v3/drizzle`), and tightening its client contract from
+`unknown` to a typed operation surface is a compile-time breaking change — a
+downstream consumer passing a loosely-typed (`unknown`-returning) client double
+will now fail `tsc`. That tightening has teeth: `operators.test-d.ts` pins it
+with a negative type-test asserting an `unknown`-returning `{ encrypt }` double
+is rejected (a positive "correctly-typed double is accepted" assertion cannot
+catch a re-erasure, since a correct value is assignable to `unknown`).
+
+Behaviour is otherwise unchanged, with one addition: the Supabase v3 bulk path
+now rejects a `null` envelope returned by `bulkEncrypt` (the restored
+`Encrypted | null` type makes that arm reachable, and a `null` would otherwise
+be `JSON.stringify`'d to the literal `"null"` and sent as a filter operand).

.changeset/supabase-v3-order-by-ope-term.md

@@ -26,7 +26,9 @@ The guard is now on the ordering FLAVOUR, not on encryption:
 - **`ore` present → rejected.** The `ob` term is an array of ORE blocks whose
   comparison needs the superuser-only operator class, which no jsonb path can
   reach. (Such a column cannot hold data on managed Postgres anyway: its domain
-  CHECK raises `ore_domain_unavailable`.)
+  CHECK raises `ore_domain_unavailable`.) ORE columns are now excluded from
+  `order()` at COMPILE time too, not only at runtime — `.order(oreColumn)` is a
+  type error, matching the rejection.
 - **neither → rejected.** Storage-only, equality-only and match-only columns
   carry no ordering term.
 
@@ -38,6 +40,8 @@ numeric and date domains, and per-character (16 hex chars each) for text, so
 lexicographic order reproduces plaintext order including the prefix case
 (`ada` < `adam`). `ope-term.integration.test.ts` pins that shape.
 
-`V3OrderableKeys` widens to match, so `order()` on an ordering column
-typechecks. `is(col, true)` is unaffected — it stays plaintext-only, and now has
-its own `V3PlaintextKeys` rather than borrowing the orderable set.
+`V3OrderableKeys` widens to admit OPE-backed ordering columns (`*_ord`,
+`text_ord`, `text_search`) while still excluding ORE (`*_ord_ore`) columns, so
+`order()` typechecks exactly where it works. `is(col, true)` is unaffected — it
+stays plaintext-only, and now has its own `V3PlaintextKeys` rather than
+borrowing the orderable set.

.github/workflows/integration-drizzle.yml

@@ -18,6 +18,12 @@ on:
     branches: [main]
     paths:
       - 'packages/stack/src/eql/v3/**'
+      # Source layers the adapter's encoding/round-trip rests on: a break here
+      # (not just under src/eql/v3) can produce wrong rows, so trigger the live
+      # suite that would catch it.
+      - 'packages/stack/src/encryption/**'
+      - 'packages/stack/src/schema/**'
+      - 'packages/schema/**'
       - 'packages/stack/integration/**'
       - 'packages/test-kit/**'
       - 'packages/cli/src/installer/**'
@@ -31,6 +37,12 @@ on:
     # Repeated verbatim: GitHub Actions does not support YAML anchors/aliases.
     paths:
       - 'packages/stack/src/eql/v3/**'
+      # Source layers the adapter's encoding/round-trip rests on: a break here
+      # (not just under src/eql/v3) can produce wrong rows, so trigger the live
+      # suite that would catch it.
+      - 'packages/stack/src/encryption/**'
+      - 'packages/stack/src/schema/**'
+      - 'packages/schema/**'
       - 'packages/stack/integration/**'
       - 'packages/test-kit/**'
       - 'packages/cli/src/installer/**'

.github/workflows/integration-supabase.yml

@@ -14,6 +14,12 @@ on:
     paths:
       - 'packages/stack/src/supabase/**'
       - 'packages/stack/src/eql/v3/**'
+      # Source layers the adapter's encoding/round-trip rests on: a break here
+      # (not just under src/supabase) can produce wrong wire output or rows, so
+      # trigger the live suite that would catch it.
+      - 'packages/stack/src/encryption/**'
+      - 'packages/stack/src/schema/**'
+      - 'packages/schema/**'
       - 'packages/stack/integration/**'
       - 'packages/test-kit/**'
       - 'packages/cli/src/installer/**'
@@ -26,6 +32,12 @@ on:
     paths:
       - 'packages/stack/src/supabase/**'
       - 'packages/stack/src/eql/v3/**'
+      # Source layers the adapter's encoding/round-trip rests on: a break here
+      # (not just under src/supabase) can produce wrong wire output or rows, so
+      # trigger the live suite that would catch it.
+      - 'packages/stack/src/encryption/**'
+      - 'packages/stack/src/schema/**'
+      - 'packages/schema/**'
       - 'packages/stack/integration/**'
       - 'packages/test-kit/**'
       - 'packages/cli/src/installer/**'

packages/stack/__tests__/clerk-provider.test.ts

@@ -0,0 +1,33 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { clerkJwtProvider } from '../integration/helpers/clerk'
+
+/**
+ * `clerkJwtProvider`'s "FAIL rather than skip" guard is the load-bearing half of
+ * the identity suites' no-silent-skip contract: if it were ever weakened to a
+ * skip or a silent `undefined`, the entire identity suite would go dark while CI
+ * stayed green. The throw is pure logic — it fires before `createClerkClient`,
+ * so it needs no live Clerk and belongs here in the credential-free unit suite
+ * rather than the integration config (which only globs `*.integration.test.ts`).
+ */
+describe('clerkJwtProvider', () => {
+  const saved = process.env.CLERK_MACHINE_TOKEN
+  const savedB = process.env.CLERK_MACHINE_TOKEN_B
+  afterEach(() => {
+    if (saved === undefined) delete process.env.CLERK_MACHINE_TOKEN
+    else process.env.CLERK_MACHINE_TOKEN = saved
+    if (savedB === undefined) delete process.env.CLERK_MACHINE_TOKEN_B
+    else process.env.CLERK_MACHINE_TOKEN_B = savedB
+  })
+
+  it('throws (does not skip) when the default machine token env var is unset', () => {
+    delete process.env.CLERK_MACHINE_TOKEN
+    expect(() => clerkJwtProvider()).toThrow(/missing CLERK_MACHINE_TOKEN/)
+  })
+
+  it('names the custom env var it was handed', () => {
+    delete process.env.CLERK_MACHINE_TOKEN_B
+    expect(() => clerkJwtProvider('CLERK_MACHINE_TOKEN_B')).toThrow(
+      /CLERK_MACHINE_TOKEN_B/,
+    )
+  })
+})

packages/stack/__tests__/drizzle-v3/operators.test-d.ts

@@ -1,7 +1,12 @@
+import type { Result } from '@byteslice/result'
 import { describe, expectTypeOf, it } from 'vitest'
 import type { EncryptionClient } from '@/encryption'
+import type { AuditConfig } from '@/encryption/operations/base-operation'
 import type { EncryptionV3 } from '@/encryption/v3'
 import { createEncryptionOperatorsV3 } from '@/eql/v3/drizzle'
+import type { EncryptionError } from '@/errors'
+import type { LockContext } from '@/identity'
+import type { Encrypted } from '@/types'
 
 /**
  * Static regression guard for M1: `createEncryptionOperatorsV3` must accept the
@@ -27,9 +32,34 @@ describe('createEncryptionOperatorsV3 - client parameter (M1)', () => {
   })
 
   it('accepts a minimal structural { encrypt } double', () => {
+    // The double now models what the real client returns: an operation whose
+    // `then` resolves a `Result<Encrypted, …>`, not `unknown`. That is the point
+    // of the un-erasure — a `{ encrypt }` returning `unknown` no longer
+    // typechecks, because the factory reads `result.data` as an `Encrypted`
+    // envelope rather than casting it.
+    type Op = {
+      withLockContext(lc: LockContext): Op
+      audit(cfg: AuditConfig): Op
+      then: PromiseLike<Result<Encrypted, EncryptionError>>['then']
+    }
     const double = {
-      encrypt: (_plaintext: never, _opts: never) => ({}) as unknown,
+      encrypt: (_plaintext: never, _opts: never) => ({}) as Op,
     }
     expectTypeOf(createEncryptionOperatorsV3).toBeCallableWith(double)
   })
+
+  it('rejects a { encrypt } double that returns `unknown` (the erasure regression)', () => {
+    // The complement of the test above, and the one with teeth: `toBeCallableWith`
+    // a correctly-typed double keeps passing even if the client contract
+    // regressed to `unknown` (a correct value is assignable to `unknown`), so it
+    // cannot catch re-erasure. This can. `encrypt` returning `unknown` must NOT
+    // satisfy the factory, whose contract is `ChainableOperation<Encrypted>`; if
+    // the erasure ever comes back, the `@ts-expect-error` goes unused and fails.
+    const erased = {
+      encrypt: (_plaintext: never, _opts: never): unknown => ({}),
+    }
+    // @ts-expect-error — `encrypt` returning `unknown` does not satisfy the
+    // factory's `ChainableOperation<Encrypted>` client contract.
+    createEncryptionOperatorsV3(erased)
+  })
 })

packages/stack/__tests__/supabase-v3-builder.test.ts

@@ -1901,4 +1901,29 @@ describe('v3 in-list term encryption batches by column', () => {
     expect(status).toBe(500)
     expect(error?.message).toMatch(/1 term(s)? for 2 value(s)?/)
   })
+
+  it('rejects a bulk response with a null envelope rather than sending "null"', async () => {
+    const supabase = createMockSupabase()
+    const encryption = createMockEncryptionClient()
+    // A length-matched response, but position 0 is a null envelope. Without the
+    // guard, `JSON.stringify(null)` would send the literal `"null"` as the `in`
+    // operand — matching whatever `"null"` encodes to instead of failing.
+    ;(
+      encryption as unknown as { bulkEncrypt: (...a: unknown[]) => unknown }
+    ).bulkEncrypt = () =>
+      operation([{ data: null }, { data: fakeEnvelope('grace', 'nickname') }])
+
+    const { error, status } = await new EncryptedQueryBuilderV3Impl(
+      'users',
+      users,
+      encryption,
+      supabase.client,
+      USERS_ALL_COLUMNS,
+    )
+      .select('id')
+      .in('nickname', ['ada', 'grace'])
+
+    expect(status).toBe(500)
+    expect(error?.message).toMatch(/null envelope at position 0/)
+  })
 })

packages/stack/__tests__/supabase-v3.test-d.ts

@@ -18,6 +18,7 @@ const users = encryptedTable('users', {
   active: types.Boolean('active'),
   nickname: types.TextEq('nickname'),
   bio: types.TextMatch('bio'),
+  score: types.IntegerOrdOre('score'),
 })
 
 type UserRow = InferPlaintext<typeof users>
@@ -130,6 +131,11 @@ describe('encryptedSupabaseV3 typed surface (with schemas)', () => {
     builder.order('nickname')
     // @ts-expect-error — bio is public.eql_v3_text_match: match only
     builder.order('bio')
+    // @ts-expect-error — score is public.eql_v3_integer_ord_ore: ORE-backed, so
+    // orderAndRange-capable but NOT sortable through a jsonb path (its `ob` term
+    // needs the superuser-only ORE opclass). Excluded at compile time to match
+    // the runtime rejection.
+    builder.order('score')
   })
 
   it('still allows order() on a plaintext row key', async () => {

packages/stack/integration/drizzle-v3/relational.integration.test.ts

@@ -28,6 +28,7 @@ import {
   eqlTypeSlug as slug,
   sortedKeysFor as sortedKeysForKit,
   typedEntries,
+  unwrapResult,
   V3_MATRIX,
 } from '@cipherstash/test-kit'
 import {
@@ -52,9 +53,15 @@ import { makeEqlV3Column } from '@/eql/v3/drizzle/column'
 
 const sqlClient = postgres(databaseUrl(), { prepare: false })
 
-const TABLE_NAME = 'protect_ci_v3_drizzle_matrix'
-const ACCOUNT_TABLE_NAME = 'protect_ci_v3_drizzle_matrix_accounts'
-const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
+// Per-run table suffix so two runs sharing a database (a persistent/reused CI
+// database, a developer's local DB, or re-enabled file parallelism) never
+// operate on the same physical table — one run's `beforeAll` DROP would
+// otherwise blow away a table another run is mid-query on. Mirrors the family
+// suites' run-scoped naming (`rows.ts` `planTable`).
+const RID = Math.random().toString(36).slice(2, 8)
+const TABLE_NAME = `protect_ci_v3_drizzle_matrix_${RID}`
+const ACCOUNT_TABLE_NAME = `protect_ci_v3_drizzle_matrix_accounts_${RID}`
+const RUN = `run-${Date.now()}-${RID}`
 const ROW_A = 'row-a'
 const ROW_B = 'row-b'
 // A third row. With only two rows every predicate can return just [A], [B],
@@ -103,7 +110,7 @@ const accountsTable = pgTable(ACCOUNT_TABLE_NAME, {
 // known at compile time, so it exercises A3 end-to-end with ZERO casts: the
 // insert takes envelope rows and the select yields `Encrypted` values ready for
 // decrypt.
-const BIGINT_TABLE_NAME = 'protect_ci_v3_drizzle_bigint'
+const BIGINT_TABLE_NAME = `protect_ci_v3_drizzle_bigint_${RID}`
 const bigintTable = pgTable(BIGINT_TABLE_NAME, {
   id: integer('id').primaryKey().generatedAlwaysAsIdentity(),
   rowKey: text('row_key').notNull(),
@@ -190,11 +197,6 @@ async function selectRowKeys(condition: SQL | undefined): Promise<string[]> {
   return rows.map((row) => row.rowKey)
 }
 
-function unwrap<T>(result: { data?: T; failure?: { message: string } }): T {
-  if (result.failure) throw new Error(result.failure.message)
-  return result.data as T
-}
-
 function encryptedInsertRows(): MatrixPlainRow[] {
   return ROWS.map((rowKey) => {
     const row: MatrixPlainRow = {
@@ -218,12 +220,13 @@ beforeAll(async () => {
     .map(([eqlType]) => `"${slug(eqlType)}" ${eqlType} NOT NULL`)
     .join(',\n      ')
 
-  // DROP, not `CREATE TABLE IF NOT EXISTS`. A table left by an earlier run keeps
-  // its old columns, so a change to the catalog silently reuses the stale schema.
-  // That bit: after the `_ord_ore` domains were filtered out of `matrixEntries`,
-  // the leftover table still carried its nine ORE columns, and on managed
-  // Postgres every INSERT raised `ore_domain_unavailable` — even leaving those
-  // columns NULL, because the domain CHECK calls a function that RAISEs.
+  // Table names are run-scoped (see RID), so DROP IF EXISTS is normally a no-op;
+  // it stays as a belt-and-braces guard against an id collision. Recreating from
+  // scratch each run also means a catalog change can never silently reuse a
+  // stale schema — the bug that bit once when the `_ord_ore` domains were
+  // filtered out of `matrixEntries` but a leftover fixed-name table kept its nine
+  // ORE columns, making every INSERT raise `ore_domain_unavailable` on managed
+  // Postgres (the domain CHECK RAISEs even for a NULL value).
   await sqlClient.unsafe(`
     DROP TABLE IF EXISTS ${TABLE_NAME}
   `)
@@ -260,7 +263,7 @@ beforeAll(async () => {
     )
   `)
 
-  const encryptedRows = unwrap(
+  const encryptedRows = unwrapResult(
     await client.bulkEncryptModels(encryptedInsertRows(), schema),
   )
   await db.insert(matrixTable).values(encryptedRows)
@@ -276,7 +279,7 @@ beforeAll(async () => {
   // ROW_B exists so the filter proofs below have a row they must EXCLUDE. On a
   // one-row table `gt(balance, 0n)` returning every row is indistinguishable
   // from it returning the right row.
-  const bigintRows = unwrap(
+  const bigintRows = unwrapResult(
     await client.bulkEncryptModels(
       [
         {
@@ -299,9 +302,11 @@ beforeAll(async () => {
 }, 120000)
 
 afterAll(async () => {
-  await sqlClient`DELETE FROM ${sqlClient(TABLE_NAME)} WHERE test_run_id = ${RUN}`
-  await sqlClient`DELETE FROM ${sqlClient(ACCOUNT_TABLE_NAME)} WHERE test_run_id = ${RUN}`
-  await sqlClient`DELETE FROM ${sqlClient(BIGINT_TABLE_NAME)} WHERE test_run_id = ${RUN}`
+  // Tables are run-scoped, so drop them outright rather than DELETE-ing rows —
+  // no other run shares them, and this leaves nothing behind on a persistent DB.
+  await sqlClient.unsafe(`DROP TABLE IF EXISTS ${TABLE_NAME}`)
+  await sqlClient.unsafe(`DROP TABLE IF EXISTS ${ACCOUNT_TABLE_NAME}`)
+  await sqlClient.unsafe(`DROP TABLE IF EXISTS ${BIGINT_TABLE_NAME}`)
   await sqlClient.end()
 }, 30000)
 
@@ -449,6 +454,25 @@ describe('v3 drizzle — relational, needle guards, pagination', () => {
     expect(rows).toEqual([ROW_B, ROW_C])
   }, 30000)
 
+  it('not(between()) negates the whole range, not just the lower bound', async () => {
+    // integer_ord: ROW_A=0, ROW_B=-42, ROW_C=2147483647. `not(between(0, 0))`
+    // must return every row whose value != 0 → ROW_B and ROW_C.
+    //
+    // `between` emits a TWO-clause conjunction, and Drizzle's passthrough `not`
+    // renders a bare `NOT <cond>`. Postgres binds NOT tighter than AND, so this
+    // only works because `v3Dialect.range` already parenthesises `(gte AND lte)`.
+    // Without those parens, `NOT gte(0) AND lte(0)` parses as
+    // `value < 0 AND value <= 0` = `value < 0` = ROW_B alone, silently dropping
+    // ROW_C. Asserting ROW_C is present is what discriminates the two — a
+    // single-bound complement would satisfy the buggy form too.
+    const rows = await selectRowKeys(
+      ops.not(
+        await ops.between(matrixColumn('public.eql_v3_integer_ord'), 0, 0),
+      ),
+    )
+    expect(rows).toEqual([ROW_B, ROW_C])
+  }, 30000)
+
   it('isNull and isNotNull work on nullable encrypted columns', async () => {
     expect(await selectRowKeys(ops.isNull(matrixTable.nullableTextEq))).toEqual(
       [ROW_A],
@@ -578,7 +602,7 @@ describe('v3 drizzle — relational, needle guards, pagination', () => {
       )
     expect(encrypted).toHaveLength(1)
 
-    const decrypted = unwrap(
+    const decrypted = unwrapResult(
       await client.decryptModel(encrypted[0], bigintSchema),
     )
     expect(decrypted.balance).toBe(BIGINT_BALANCE)