Skip to content

Commit 73d5a65

Browse files
authored
test(vertical): composition eval suite over the assembled chat modules (#188) (#193)
* fix(chat-store): make bulk-delete workspace access checks deterministic bulkDeleteThreads iterated workspaces in DB row order of random hex thread ids, so which workspace's denial surfaced first varied per run — the unit test asserting the check sequence was flaky (caught by the vertical composition suite runs). Sort the distinct workspace ids before checking; fail-closed semantics are unchanged. * fix(interactions): type the persisted-part codec returns interactionToPersistedPart/noticePart returned Record<string, unknown>, so every product pushing them into a ChatMessagePart[] transcript needed a cast, and the 'byte-matches' claim between the codecs and chat-store's ChatInteractionPart/ChatNoticePart was prose only. Return the precise InteractionPersistedPart/NoticePersistedPart shapes (type aliases, so Record<string, unknown> consumers stay source-compatible) and enforce mutual assignability with the stored part types at compile time in chat-store/parts.ts. * fix(sandbox): make the provision-gate profile section structural ProvisionPayloadSections claimed to be structural but typed backend.profile as the SDK's AgentProfile, so even this module's own call site cast into its own gate type, and the SDK's string profile ref was rejected. Type the slice the gate actually reads (ProvisionProfileSection: resources.files) plus the string ref arm, and drop the internal cast. * test(vertical): composition eval suite over the assembled chat modules A mini chat app assembled ONLY from the shipped subpaths — createAppAuth (better-auth memoryAdapter) + createChatTables/createChatStore (:memory: better-sqlite3) + the #190 turn-boundary gates + a fake sandbox producer emitting canonical sidecar part streams pumped through /stream's normalizer into both the NDJSON client lane and the persisted parts lane + createInteractionAnswerRoute over a fake sidecar broker the producer genuinely blocks on. 15 scenario evals assert user-visible outcomes only: full turn persists parts + token receipts; failed turn keeps the user message and surfaces the error; interaction ask blocks the turn, rejects invalid answers fail-closed, unblocks on answer, and persists the answered card; re-emitted duplicate questions render one card and one answer resolves both; cross-workspace reads/answers/lists/deletes are denied with zero side effects; the three #190 gates fire client-visible 400s with their breakdown messages before any row is written. This suite is the composition gate for #189/#190/#191/#192: a subpath change that passes its unit tests but breaks assembly fails here. * fix(ci): raise the tsup d.ts worker heap to 8192MB The d.ts worker's memory footprint grows with the subpath count; since the #191/#192 merges every main CI run (and the next publish) OOMs at the 6144MB job heap during the install-time prepare build. 8192 matches the documented local build requirement; public-repo runners have 16GB.
1 parent 7014476 commit 73d5a65

9 files changed

Lines changed: 1196 additions & 11 deletions

File tree

.github/workflows/ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ jobs:
1111
env:
1212
# tsup generates .d.ts for ~38 entries in a worker thread (the `prepare`
1313
# script runs during install, and again in the build step); that worker
14-
# needs ~4.7GB and OOMs at the default limit. The worker inherits
14+
# needed ~4.7GB and has grown with the subpath count (OOMed at 6144 after the #191/#192 merges); 8192 matches the documented local build requirement. The worker inherits
1515
# NODE_OPTIONS, so set the heap once at the job level.
16-
NODE_OPTIONS: --max-old-space-size=6144
16+
NODE_OPTIONS: --max-old-space-size=8192
1717
steps:
1818
- uses: actions/checkout@v7
1919
- uses: pnpm/action-setup@v6

.github/workflows/publish.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
contents: write # push the version-bump commit + tag
3535
id-token: write # OIDC trusted publish + Sigstore provenance
3636
env:
37-
NODE_OPTIONS: --max-old-space-size=6144
37+
NODE_OPTIONS: --max-old-space-size=8192
3838
steps:
3939
- uses: actions/checkout@v7
4040
with:
@@ -93,7 +93,7 @@ jobs:
9393
# ERR_WORKER_OUT_OF_MEMORY on the hosted runner. The worker inherits
9494
# NODE_OPTIONS, so a job-level heap covers install (prepare), test, and
9595
# build in one place.
96-
NODE_OPTIONS: --max-old-space-size=6144
96+
NODE_OPTIONS: --max-old-space-size=8192
9797
steps:
9898
- uses: actions/checkout@v7
9999
- uses: pnpm/action-setup@v6
@@ -129,7 +129,7 @@ jobs:
129129
env:
130130
# Same DTS-worker heap need: this job runs `pnpm install` (prepare → tsup)
131131
# and `pnpm run build` before publishing.
132-
NODE_OPTIONS: --max-old-space-size=6144
132+
NODE_OPTIONS: --max-old-space-size=8192
133133
permissions:
134134
contents: read
135135
id-token: write

src/chat-store/parts.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,9 @@ import type { Part as HarnessWirePart } from '@tangle-network/agent-interface'
4444
import type {
4545
ChatInteractionField,
4646
ChatInteractionStatus,
47+
InteractionPersistedPart,
4748
NoticeKind,
49+
NoticePersistedPart,
4850
} from '../web-react/chat-interactions'
4951

5052
/** Start/end wall-clock millis, as normalized by `/stream`'s `normalizeTime`. */
@@ -168,6 +170,15 @@ export interface ChatNoticePart {
168170
text: string
169171
}
170172

173+
// The "byte-matches" claims above, enforced at compile time: the interaction
174+
// contract's codec output types and the stored part types must stay mutually
175+
// assignable, so a codec field added on one side without the other fails here.
176+
type MutuallyAssignable<A extends B, B> = A
177+
type _CodecEmitsStorableInteractionPart = MutuallyAssignable<InteractionPersistedPart, ChatInteractionPart>
178+
type _StoredInteractionPartFeedsCodec = MutuallyAssignable<ChatInteractionPart, InteractionPersistedPart>
179+
type _CodecEmitsStorableNoticePart = MutuallyAssignable<NoticePersistedPart, ChatNoticePart>
180+
type _StoredNoticePartFeedsCodec = MutuallyAssignable<ChatNoticePart, NoticePersistedPart>
181+
171182
export type ChatMessagePart =
172183
| ChatTextPart
173184
| ChatReasoningPart

src/chat-store/store.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,9 @@ export function createChatStore<TTables extends ChatTables>(
230230

231231
// Access is verified once per workspace the ids touch. Fail-closed: one
232232
// inaccessible workspace rejects the whole request before any delete.
233-
const workspaceIds = [...new Set(rows.map((row) => row.workspaceId))]
233+
// Sorted so the check order (and therefore which denial surfaces) is
234+
// deterministic — row order follows random hex ids and varies per run.
235+
const workspaceIds = [...new Set(rows.map((row) => row.workspaceId))].sort()
234236
for (const workspaceId of workspaceIds) {
235237
await assertAccess(workspaceId)
236238
}

src/interactions/contract.ts

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -226,9 +226,35 @@ export function noticePartKey(id: string): string {
226226

227227
export type NoticeKind = 'warning' | 'auto-declined'
228228

229+
/**
230+
* Persisted-part shapes the codecs below produce — the SAME rows
231+
* `/chat-store`'s `ChatInteractionPart`/`ChatNoticePart` store, typed at the
232+
* source so a product pushing them into a `ChatMessagePart[]` transcript needs
233+
* no cast. Type aliases (not interfaces) on purpose: the implicit index
234+
* signature keeps them assignable to the `Record<string, unknown>` these
235+
* codecs previously returned, so existing consumers stay source-compatible.
236+
*/
237+
export type InteractionPersistedPart = {
238+
type: 'interaction'
239+
id: string
240+
kind: string
241+
title: string
242+
body?: string
243+
answerSpec: { fields: ChatInteractionField[] }
244+
status: ChatInteractionStatus
245+
cancelReason?: string
246+
}
247+
248+
export type NoticePersistedPart = {
249+
type: 'notice'
250+
id: string
251+
noticeKind: NoticeKind
252+
text: string
253+
}
254+
229255
/** Builds the persisted/streamed `notice` part — a one-line transcript notice
230256
* explaining an out-of-band event (warning, auto-declined interaction). */
231-
export function noticePart(noticeKind: NoticeKind, id: string, text: string): Record<string, unknown> {
257+
export function noticePart(noticeKind: NoticeKind, id: string, text: string): NoticePersistedPart {
232258
return { type: 'notice', id, noticeKind, text }
233259
}
234260

@@ -249,7 +275,7 @@ export function interactionToPersistedPart(
249275
request: InteractionRequestWire,
250276
status: ChatInteractionStatus,
251277
cancelReason?: string,
252-
): Record<string, unknown> {
278+
): InteractionPersistedPart {
253279
return {
254280
type: 'interaction',
255281
id: request.id,

src/sandbox/index.ts

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,12 +1035,22 @@ function utf8ByteLength(value: unknown): number {
10351035
.byteLength
10361036
}
10371037

1038+
/** Structural slice of the profile the payload gate reads: it only measures
1039+
* the profile's serialized size and names `resources.files` in the breakdown,
1040+
* so callers composing a payload outside the SDK (products, tests) can pass a
1041+
* plain object without casting through `AgentProfile`. */
1042+
export interface ProvisionProfileSection {
1043+
resources?: { files?: readonly unknown[] }
1044+
}
1045+
10381046
/** The provision-payload sections the size gates need to see. Structural so
10391047
* the gate is testable without the SDK's (unexported) create-payload type. */
10401048
export interface ProvisionPayloadSections {
10411049
env?: Record<string, string>
10421050
secrets?: readonly string[]
1043-
backend?: { profile?: AgentProfile }
1051+
/** `profile` may also be a named-profile string ref (the SDK's
1052+
* `BackendConfig` union) — a string ref is tiny and has no files channel. */
1053+
backend?: { profile?: string | ProvisionProfileSection }
10441054
}
10451055

10461056
/**
@@ -1053,7 +1063,7 @@ export function assertProvisionPayloadWithinCap(payload: ProvisionPayloadSection
10531063
const total = utf8ByteLength(payload)
10541064
if (total <= PROVISION_PAYLOAD_MAX_BYTES) return
10551065
const profile = payload.backend?.profile
1056-
const files = profile?.resources?.files ?? []
1066+
const files = (typeof profile === 'string' ? undefined : profile?.resources?.files) ?? []
10571067
const breakdown =
10581068
`profile=${utf8ByteLength(profile ?? null)}B ` +
10591069
`(files=${utf8ByteLength(files)}B), ` +
@@ -1549,7 +1559,9 @@ export async function ensureWorkspaceSandbox(
15491559
// provision body can never produce a working sandbox — fail loud here,
15501560
// before the POST, with the offending section named.
15511561
assertEnvWithinLimits(env)
1552-
assertProvisionPayloadWithinCap(payload as ProvisionPayloadSections)
1562+
// `?? {}` only narrows the SDK parameter's `| undefined`; the literal above
1563+
// is always defined. The structural sections type needs no cast.
1564+
assertProvisionPayloadWithinCap(payload ?? {})
15531565

15541566
let box = await client.create(payload)
15551567

tests/vertical/fake-sidecar.ts

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
/**
2+
* Fake sandbox sidecar for the vertical composition suite: ONE in-memory
3+
* interaction registry that BOTH halves of `/interactions` talk to —
4+
*
5+
* - the producer half: the fake sandbox turn registers an ask via `ask()`
6+
* and blocks on the returned promise (the broker semantics the real
7+
* sidecar's InteractionBroker has);
8+
* - the answer half: `createInteractionAnswerRoute` reaches the same
9+
* registry through `connection.fetchImpl` (GET list / POST respond), so
10+
* answering through the route genuinely releases the blocked turn.
11+
*
12+
* Answer validation is fail-closed like the real sidecar: an `accepted`
13+
* outcome must satisfy the ask's `answerSpec` (a select without `allowCustom`
14+
* only accepts listed option values; required fields must be present) or the
15+
* POST fails 400 INVALID_INTERACTION_ANSWER. Unknown ids fail 404.
16+
*/
17+
18+
import type {
19+
ChatSelectField,
20+
InteractionData,
21+
InteractionRequestWire,
22+
SidecarInteractionsConnection,
23+
} from '../../src/interactions/index'
24+
25+
export interface InteractionResolutionRecord {
26+
outcome: 'accepted' | 'declined'
27+
data?: InteractionData
28+
}
29+
30+
interface Waiter {
31+
resolve: (resolution: InteractionResolutionRecord) => void
32+
promise: Promise<InteractionResolutionRecord>
33+
}
34+
35+
function validateAcceptedAnswer(request: InteractionRequestWire, data: InteractionData | undefined): string | null {
36+
for (const field of request.answerSpec.fields) {
37+
const value = data?.[field.name]
38+
if (value === undefined) {
39+
if (field.required) return `missing required field ${field.name}`
40+
continue
41+
}
42+
if (field.type === 'select') {
43+
const select = field as ChatSelectField
44+
if (select.allowCustom === true) continue
45+
const values = Array.isArray(value) ? value : [String(value)]
46+
const allowed = new Set(select.options.map((option) => option.value))
47+
for (const candidate of values) {
48+
if (!allowed.has(String(candidate))) return `value ${String(candidate)} is not a listed option for ${field.name}`
49+
}
50+
}
51+
}
52+
return null
53+
}
54+
55+
export interface FakeSidecarSession {
56+
/** Producer half: register an ask and get the broker promise the turn
57+
* blocks on. */
58+
ask(request: InteractionRequestWire): Promise<InteractionResolutionRecord>
59+
/** Resolves when every currently-outstanding ask has been answered. */
60+
waitAll(): Promise<Map<string, InteractionResolutionRecord>>
61+
/** Asks still waiting for an answer (the sidecar registry view). */
62+
outstandingIds(): string[]
63+
/** How the answer route reaches this session. */
64+
connection: SidecarInteractionsConnection
65+
/** Every HTTP call the answer route made, for hostile-integrator asserts. */
66+
calls: Array<{ method: string; body?: Record<string, unknown> }>
67+
}
68+
69+
export function createFakeSidecarSession(sessionId = 'session-vertical'): FakeSidecarSession {
70+
const outstanding = new Map<string, InteractionRequestWire>()
71+
const waiters = new Map<string, Waiter>()
72+
const resolutions = new Map<string, InteractionResolutionRecord>()
73+
const calls: FakeSidecarSession['calls'] = []
74+
75+
function ask(request: InteractionRequestWire): Promise<InteractionResolutionRecord> {
76+
outstanding.set(request.id, request)
77+
let resolve!: Waiter['resolve']
78+
const promise = new Promise<InteractionResolutionRecord>((res) => {
79+
resolve = res
80+
})
81+
waiters.set(request.id, { resolve, promise })
82+
return promise
83+
}
84+
85+
const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => {
86+
const method = init?.method ?? 'GET'
87+
const body = init?.body ? (JSON.parse(String(init.body)) as Record<string, unknown>) : undefined
88+
calls.push({ method, ...(body ? { body } : {}) })
89+
void input
90+
91+
if (method === 'GET') {
92+
return Response.json({ data: { interactions: [...outstanding.values()] } })
93+
}
94+
95+
const id = String(body?.id ?? '')
96+
const request = outstanding.get(id)
97+
if (!request) {
98+
return new Response(
99+
JSON.stringify({ error: { code: 'NOT_FOUND', message: 'interaction not found' } }),
100+
{ status: 404 },
101+
)
102+
}
103+
const outcome = body?.outcome === 'declined' ? 'declined' : 'accepted'
104+
const data = body?.data as InteractionData | undefined
105+
if (outcome === 'accepted') {
106+
const invalid = validateAcceptedAnswer(request, data)
107+
if (invalid) {
108+
return new Response(
109+
JSON.stringify({ error: { code: 'INVALID_INTERACTION_ANSWER', message: invalid } }),
110+
{ status: 400 },
111+
)
112+
}
113+
}
114+
outstanding.delete(id)
115+
const resolution: InteractionResolutionRecord = { outcome, ...(data ? { data } : {}) }
116+
resolutions.set(id, resolution)
117+
waiters.get(id)?.resolve(resolution)
118+
waiters.delete(id)
119+
return Response.json({ data: { ok: true } })
120+
}) as typeof fetch
121+
122+
return {
123+
ask,
124+
async waitAll() {
125+
await Promise.all([...waiters.values()].map((waiter) => waiter.promise))
126+
return new Map(resolutions)
127+
},
128+
outstandingIds: () => [...outstanding.keys()],
129+
connection: {
130+
runtimeUrl: 'https://box.vertical.test/runtime',
131+
authToken: 'tc_vertical-sidecar-bearer',
132+
sessionId,
133+
fetchImpl,
134+
},
135+
calls,
136+
}
137+
}

0 commit comments

Comments
 (0)