diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 9ab747b15..3bd1d5d2a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -123,6 +123,8 @@ jobs: packages/ui/src/stores/session-metadata.test.ts packages/ui/src/stores/session-pagination.test.ts packages/ui/src/stores/workspace-list-reconciliation-fence.test.ts + packages/ui/src/stores/workflows.test.ts + packages/opencode-plugin/plugin/lib/workflows.test.ts - name: Test restore ownership integration run: >- diff --git a/packages/opencode-plugin/plugin/codenomad.ts b/packages/opencode-plugin/plugin/codenomad.ts index 61d1827f0..cee264840 100644 --- a/packages/opencode-plugin/plugin/codenomad.ts +++ b/packages/opencode-plugin/plugin/codenomad.ts @@ -1,17 +1,19 @@ import type { PluginInput } from "@opencode-ai/plugin" import { createCodeNomadClient, getCodeNomadConfig } from "./lib/client.js" import { createBackgroundProcessTools } from "./lib/background-process.js" +import { createWorkflowTools } from "./lib/workflows.js" let voiceModeEnabled = false export async function CodeNomadPlugin(input: PluginInput): Promise<{ - tool: ReturnType + tool: ReturnType & ReturnType "chat.message": CodeNomadChatMessageHook event: CodeNomadEventHook }> { const config = getCodeNomadConfig() const client = createCodeNomadClient(config) const backgroundProcessTools = createBackgroundProcessTools(config, { baseDir: input.directory }) + const workflowTools = createWorkflowTools(config) await client.startEvents((event) => { if (event.type === "codenomad.ping") { @@ -33,6 +35,7 @@ export async function CodeNomadPlugin(input: PluginInput): Promise<{ return { tool: { ...backgroundProcessTools, + ...workflowTools, }, async "chat.message"(_input: { sessionID: string }, output: { message: { system?: string } }) { if (!voiceModeEnabled) { diff --git a/packages/opencode-plugin/plugin/lib/request.test.ts b/packages/opencode-plugin/plugin/lib/request.test.ts new file mode 100644 index 000000000..0a3e5f9a6 --- /dev/null +++ b/packages/opencode-plugin/plugin/lib/request.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict" +import { createServer } from "node:http" +import { test } from "node:test" + +import { createCodeNomadRequester } from "./request" + +test("plugin requests use the distinct callback capability", async () => { + let authorization: string | undefined + const server = createServer((request, response) => { + authorization = request.headers.authorization + response.writeHead(200, { Connection: "close" }).end() + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + try { + const address = server.address() + assert.ok(address && typeof address === "object") + const requester = createCodeNomadRequester({ + instanceId: "workspace", + baseUrl: `http://127.0.0.1:${address.port}`, + callbackToken: "workspace-callback", + }) + + await requester.requestVoid("/event", { method: "POST" }) + + assert.equal(authorization, "Bearer workspace-callback") + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } +}) + +test("plugin responses use null bodies when HTTP forbids response content", async () => { + const server = createServer((request, response) => { + const status = Number(new URL(request.url ?? "/", "http://localhost").pathname.slice(1)) || 200 + response.writeHead(status, { Connection: "close" }).end("ignored") + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + try { + const address = server.address() + assert.ok(address && typeof address === "object") + const requester = createCodeNomadRequester({ + instanceId: "workspace", + baseUrl: `http://127.0.0.1:${address.port}`, + callbackToken: "workspace-callback", + }) + + for (const status of [204, 205, 304]) { + const response = await requester.fetch(`http://127.0.0.1:${address.port}/${status}`) + assert.equal(response.status, status) + assert.equal(response.body, null) + } + assert.equal(await requester.requestJson(`http://127.0.0.1:${address.port}/205`), undefined) + const head = await requester.fetch(`http://127.0.0.1:${address.port}/200`, { method: "HEAD" }) + assert.equal(head.body, null) + assert.equal(await requester.requestJson(`http://127.0.0.1:${address.port}/200`, { method: "HEAD" }), undefined) + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } +}) diff --git a/packages/opencode-plugin/plugin/lib/request.ts b/packages/opencode-plugin/plugin/lib/request.ts index 5025a5013..0ab62bc5d 100644 --- a/packages/opencode-plugin/plugin/lib/request.ts +++ b/packages/opencode-plugin/plugin/lib/request.ts @@ -10,12 +10,14 @@ export type PluginEvent = { export type CodeNomadConfig = { instanceId: string baseUrl: string + callbackToken: string } export function getCodeNomadConfig(): CodeNomadConfig { return { instanceId: requireEnv("CODENOMAD_INSTANCE_ID"), baseUrl: requireEnv("CODENOMAD_BASE_URL"), + callbackToken: requireEnv("CODENOMAD_CALLBACK_TOKEN"), } } @@ -23,7 +25,7 @@ export function createCodeNomadRequester(config: CodeNomadConfig) { const rawBaseUrl = (config.baseUrl ?? "").trim() const baseUrl = rawBaseUrl.replace(/\/+$/, "") const pluginBase = `${baseUrl}/workspaces/${encodeURIComponent(config.instanceId)}/plugin` - const authorization = buildInstanceAuthorizationHeader() + const authorization = `Bearer ${config.callbackToken}` const buildUrl = (path: string) => { if (path.startsWith("http://") || path.startsWith("https://")) { @@ -60,7 +62,7 @@ export function createCodeNomadRequester(config: CodeNomadConfig) { throw new Error(message || `Request failed with ${response.status}`) } - if (response.status === 204) { + if ((init?.method ?? "GET").toUpperCase() === "HEAD" || response.status === 204 || response.status === 205) { return undefined as T } @@ -117,6 +119,7 @@ async function nodeFetch( ...(isHttps ? { rejectUnauthorized: tls.rejectUnauthorized } : {}), }, (res) => { + const status = res.statusCode ?? 0 const responseHeaders = new Headers() for (const [key, value] of Object.entries(res.headers)) { if (value === undefined) continue @@ -127,9 +130,10 @@ async function nodeFetch( } } - // Convert Node stream -> Web ReadableStream for Response. - const webBody = Readable.toWeb(res) as unknown as ReadableStream - resolve(new Response(webBody, { status: res.statusCode ?? 0, headers: responseHeaders })) + const bodyForbidden = method === "HEAD" || status === 204 || status === 205 || status === 304 + if (bodyForbidden) res.resume() + const webBody = bodyForbidden ? null : Readable.toWeb(res) as unknown as ReadableStream + resolve(new Response(webBody, { status, headers: responseHeaders })) }, ) @@ -185,13 +189,6 @@ function requireEnv(key: string): string { return value } -function buildInstanceAuthorizationHeader(): string { - const username = requireEnv("OPENCODE_SERVER_USERNAME") - const password = requireEnv("OPENCODE_SERVER_PASSWORD") - const token = Buffer.from(`${username}:${password}`, "utf8").toString("base64") - return `Basic ${token}` -} - function normalizeHeaders(headers: HeadersInit | undefined): Record { const output: Record = {} if (!headers) return output diff --git a/packages/opencode-plugin/plugin/lib/workflows.test.ts b/packages/opencode-plugin/plugin/lib/workflows.test.ts new file mode 100644 index 000000000..ec1483647 --- /dev/null +++ b/packages/opencode-plugin/plugin/lib/workflows.test.ts @@ -0,0 +1,164 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { + createWorkflowTools, + describeWorkflowDefinition, + describeWorkflowDefinitions, + describeWorkflowDetails, + parseWorkflowInputs, +} from "./workflows.js" + +const run = { + id: "run", + objective: "Ship it", + status: "running" as const, + steps: [{ id: "build", title: "Build", status: "pending" }], +} + +test("workflow review messaging handles final and truncated gates", () => { + const waiting = { + ...run, + status: "waiting_for_review" as const, + pendingReviewStepId: "build", + steps: [{ + id: "build", + title: "Build", + status: "completed", + sessionId: "session-1", + output: "partial", + outputTruncated: true, + }], + } + const details = describeWorkflowDetails(waiting) + assert.match(details, /continue or complete/) + assert.match(details, /truncated/) + assert.match(details, /session-1/) +}) + +test("dynamic workflow details include execution progress, usage, statuses, and gate guidance", () => { + const dynamic = { + id: "dynamic-run", + objective: "Deploy", + status: "waiting_for_input" as const, + definitionId: "deploy", + definitionRevision: 3, + steps: [], + executionNodes: [ + { instanceKey: "plan", status: "completed", sessionIds: ["session-1"] }, + { instanceKey: "environment", status: "waiting" }, + ], + pendingGate: { + executionNodeId: "gate-execution-id", + gate: "input" as const, + prompt: "Choose an environment", + inputSchema: { type: "string", enum: ["staging", "production"] }, + }, + usage: { + cost: 0.25, + tokens: 120, + inputTokens: 70, + outputTokens: 40, + reasoningTokens: 10, + cacheReadTokens: 5, + cacheWriteTokens: 0, + }, + } + const message = describeWorkflowDetails(dynamic) + assert.match(message, /Status: waiting_for_input/) + assert.match(message, /Execution nodes: 2 total \(completed: 1, waiting: 1\)/) + assert.match(message, /Usage: 120 tokens/) + assert.match(message, /Choose an environment/) + assert.match(message, /Expected input schema/) + assert.match(message, /human in the CodeNomad UI/) + assert.match(message, /cannot answer this gate/) + assert.match(message, /session-1/) + + const approval = describeWorkflowDetails({ + ...dynamic, + status: "waiting_for_review", + pendingGate: { ...dynamic.pendingGate, gate: "approval" }, + }) + assert.match(approval, /cannot approve this gate/) + + for (const status of ["pausing", "paused", "recovery_required"] as const) { + assert.match(describeWorkflowDetails({ ...dynamic, status, pendingGate: undefined }), new RegExp(`Status: ${status}`)) + assert.match(describeWorkflowDetails({ ...dynamic, status, pendingGate: undefined }), /CodeNomad UI/) + } +}) + +test("saved workflow definition messages expose current revision and canonical definition", () => { + const definition = { + id: "deploy", + revision: 3, + definition: { name: "Deploy", description: "Deploy safely" }, + canonical: '{"version":1,"id":"deploy"}', + } + assert.match(describeWorkflowDefinitions([definition]), /deploy \| revision 3 \| Deploy \| Deploy safely/) + assert.match(describeWorkflowDefinition(definition), /Canonical definition:\n\{"version":1/) + assert.equal(describeWorkflowDefinitions([]), "No saved CodeNomad workflow definitions found.") +}) + +test("saved workflow inputs require a JSON object", () => { + assert.deepEqual(parseWorkflowInputs('{"environment":"staging"}'), { environment: "staging" }) + assert.equal(parseWorkflowInputs(), undefined) + assert.throws(() => parseWorkflowInputs("not-json"), /valid JSON/) + assert.throws(() => parseWorkflowInputs("[]"), /JSON object/) + assert.throws(() => parseWorkflowInputs("null"), /JSON object/) + let nested: unknown = true + for (let depth = 0; depth < 21; depth++) nested = { nested } + assert.throws(() => parseWorkflowInputs(JSON.stringify(nested)), /deeply nested/) + assert.throws(() => parseWorkflowInputs(JSON.stringify({ values: Array(50_001).fill(null) })), /too many values/) + assert.throws(() => parseWorkflowInputs(JSON.stringify({ value: "é".repeat(128_001) })), /too large/) +}) + +test("saved definition tools use read-only definition routes and current-revision start payload", async () => { + const calls: Array<{ path: string; init?: RequestInit }> = [] + const definition = { + id: "deploy_flow", + revision: 3, + definition: { name: "Deploy" }, + canonical: '{"version":1}', + } + const requester = { + async requestJson(path: string, init?: RequestInit): Promise { + calls.push({ path, init }) + if (path.endsWith("/start")) { + return { + id: "run-id", + objective: "Ship it", + status: "running", + definitionId: "deploy_flow", + definitionRevision: 3, + steps: [], + executionNodes: [], + } as T + } + if (path === "/workflow-definitions") return { definitions: [definition] } as T + return definition as T + }, + } + const tools = createWorkflowTools({ instanceId: "workspace", baseUrl: "http://localhost", callbackToken: "callback" }, requester) + assert.equal("start_codenomad_workflow" in tools, false) + + await tools.list_codenomad_workflow_definitions.execute({}, {} as never) + await tools.get_codenomad_workflow_definition.execute({ definition_id: "deploy_flow" }, {} as never) + const started = await tools.start_codenomad_workflow_definition.execute({ + definition_id: "deploy_flow", + objective: "Ship it", + inputs_json: '{"environment":"production"}', + }, { sessionID: "session-1" } as never) + + assert.deepEqual(calls.map((call) => call.path), [ + "/workflow-definitions", + "/workflow-definitions/deploy_flow", + "/workflow-definitions/deploy_flow/start", + ]) + assert.equal(calls[2]?.init?.method, "POST") + assert.deepEqual(JSON.parse(String(calls[2]?.init?.body)), { + objective: "Ship it", + inputs: { environment: "production" }, + }) + assert.match(started, /current saved definition revision/) + assert.doesNotMatch(String(calls[2]?.init?.body), /definitionRevision/) + assert.doesNotMatch(String(calls[2]?.init?.body), /initiatorSessionId/) +}) diff --git a/packages/opencode-plugin/plugin/lib/workflows.ts b/packages/opencode-plugin/plugin/lib/workflows.ts new file mode 100644 index 000000000..2bd76350b --- /dev/null +++ b/packages/opencode-plugin/plugin/lib/workflows.ts @@ -0,0 +1,287 @@ +import { tool } from "@opencode-ai/plugin/tool" +import { createCodeNomadRequester, type CodeNomadConfig } from "./request.js" + +type WorkflowStatus = + | "running" + | "pausing" + | "paused" + | "waiting_for_review" + | "waiting_for_input" + | "completed" + | "failed" + | "cancelled" + | "interrupted" + | "recovery_required" + +type WorkflowUsage = { + cost: number + tokens: number + inputTokens: number + outputTokens: number + reasoningTokens: number + cacheReadTokens: number + cacheWriteTokens: number +} + +type WorkflowRun = { + id: string + objective: string + status: WorkflowStatus + error?: string + pendingReviewStepId?: string + definitionId?: string + definitionRevision?: number + steps: Array<{ + id: string + title: string + status: string + sessionId?: string + output?: unknown + outputTruncated?: boolean + }> + executionNodes?: Array<{ + instanceKey: string + status: string + sessionIds?: string[] + error?: string + outputTruncated?: boolean + usage?: WorkflowUsage + }> + pendingGate?: { + executionNodeId: string + gate: "approval" | "input" + prompt: string + inputSchema?: Record + } + usage?: WorkflowUsage +} + +type WorkflowDefinitionRecord = { + id: string + revision: number + definition: { + name: string + description?: string + } + canonical: string +} + +const JSON_VALUE_BYTES_LIMIT = 256_000 +const JSON_VALUE_DEPTH_LIMIT = 20 +const JSON_VALUE_COUNT_LIMIT = 50_000 + +function summarize(run: WorkflowRun) { + const dynamic = Boolean(run.definitionId || run.executionNodes) + const progress = dynamic + ? summarizeExecution(run) + : run.steps.map((step) => `${step.title}: ${step.status}${step.sessionId ? ` (${step.sessionId})` : ""}`).join("\n") + const guidance = statusGuidance(run.status) + return [ + `Workflow ${run.id}`, + `Status: ${run.status}`, + progress, + guidance, + dynamic ? describePendingGate(run) : "", + run.error ? `Error: ${run.error}` : "", + ].filter(Boolean).join("\n") +} + +function details(run: WorkflowRun) { + if (run.definitionId || run.executionNodes) { + const nodes = (run.executionNodes ?? []).map((node) => { + const sessions = node.sessionIds?.length ? ` (${node.sessionIds.join(", ")})` : "" + const usage = node.usage ? ` | ${node.usage.tokens} tokens, cost ${node.usage.cost}` : "" + const error = node.error ? ` | error: ${node.error}` : "" + const truncated = node.outputTruncated ? " | output truncated; inspect the session in CodeNomad" : "" + return `${node.instanceKey}: ${node.status}${sessions}${usage}${error}${truncated}` + }) + return [summarize(run), nodes.length ? `Execution details:\n${nodes.join("\n")}` : ""].filter(Boolean).join("\n") + } + const reviewed = run.steps.find((step) => step.id === run.pendingReviewStepId) + const output = reviewed?.output === undefined ? "" : `\nPending review:\n${JSON.stringify(reviewed.output, null, 2)}` + const truncated = reviewed?.outputTruncated + ? `\nThis output is truncated. Review the full generated session${reviewed.sessionId ? ` ${reviewed.sessionId}` : ""} before approval.` + : "" + return `${summarize(run)}${output}${truncated}` +} + +export const describeWorkflowDetails = details + +export function parseWorkflowInputs(value?: string): Record | undefined { + if (value === undefined) return undefined + if (Buffer.byteLength(value, "utf8") > JSON_VALUE_BYTES_LIMIT) { + throw new Error("Workflow inputs are too large.") + } + let parsed: unknown + try { + parsed = JSON.parse(value) + } catch { + throw new Error("Workflow inputs must be valid JSON.") + } + if (parsed === null || Array.isArray(parsed) || typeof parsed !== "object") { + throw new Error("Workflow inputs must be a JSON object.") + } + const issue = inspectJsonValue(parsed) + if (issue) throw new Error(`Workflow inputs ${issue}.`) + return parsed as Record +} + +function inspectJsonValue(input: unknown): string | undefined { + const pending: Array<{ value: unknown; depth: number }> = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let count = 0 + + while (pending.length) { + const { value, depth } = pending.pop()! + if (++count > JSON_VALUE_COUNT_LIMIT) return "contain too many values" + if (depth > JSON_VALUE_DEPTH_LIMIT) return "are too deeply nested" + if (value === null || typeof value === "string" || typeof value === "boolean") continue + if (typeof value === "number" && Number.isFinite(value)) continue + if (!value || typeof value !== "object") return "must contain only JSON values" + if (seen.has(value)) return "must not contain cycles or aliases" + seen.add(value) + + if (Array.isArray(value)) { + const keys = Object.keys(value) + if (keys.length !== value.length || keys.some((key, index) => key !== String(index))) { + return "must contain only plain JSON arrays" + } + for (let index = value.length - 1; index >= 0; index--) { + pending.push({ value: value[index], depth: depth + 1 }) + } + continue + } + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return "must contain only plain JSON objects" + for (const child of Object.values(value)) pending.push({ value: child, depth: depth + 1 }) + } +} + +function summarizeExecution(run: WorkflowRun) { + const nodes = run.executionNodes ?? [] + const counts = new Map() + for (const node of nodes) counts.set(node.status, (counts.get(node.status) ?? 0) + 1) + const statuses = [...counts].map(([status, count]) => `${status}: ${count}`).join(", ") + const identity = `Definition: ${run.definitionId ?? "unknown"}${run.definitionRevision ? ` revision ${run.definitionRevision}` : ""}` + const execution = `Execution nodes: ${nodes.length} total${statuses ? ` (${statuses})` : ""}` + const usage = run.usage + ? `Usage: ${run.usage.tokens} tokens (input ${run.usage.inputTokens}, output ${run.usage.outputTokens}, reasoning ${run.usage.reasoningTokens}, cache read ${run.usage.cacheReadTokens}, cache write ${run.usage.cacheWriteTokens}); cost ${run.usage.cost}` + : "Usage: unavailable" + return `${identity}\n${execution}\n${usage}` +} + +function statusGuidance(status: WorkflowStatus) { + if (status === "waiting_for_review") return "Human approval is required in the CodeNomad UI before the workflow can continue or complete; plugin credentials cannot approve it." + if (status === "waiting_for_input") return "Human input is required in the CodeNomad UI; plugin credentials cannot answer it." + if (status === "pausing") return "The workflow is pausing. Monitor it in the CodeNomad UI." + if (status === "paused") return "The workflow is paused. Only a human can resume it in the CodeNomad UI." + if (status === "recovery_required") return "Recovery confirmation is required from a human in the CodeNomad UI; plugin credentials cannot confirm recovery." + return "" +} + +function describePendingGate(run: WorkflowRun) { + const gate = run.pendingGate + if (!gate) return "" + const action = gate.gate === "approval" ? "Approve or reject" : "Provide the requested input" + const restriction = gate.gate === "approval" ? "cannot approve this gate" : "cannot answer this gate" + const schema = gate.inputSchema ? `\nExpected input schema:\n${JSON.stringify(gate.inputSchema, null, 2)}` : "" + return `Pending ${gate.gate} gate (${gate.executionNodeId}): ${gate.prompt}${schema}\n${action} as a human in the CodeNomad UI; plugin credentials ${restriction}.` +} + +export function describeWorkflowDefinitions(definitions: WorkflowDefinitionRecord[]) { + if (definitions.length === 0) return "No saved CodeNomad workflow definitions found." + return definitions.map((record) => [ + `${record.id} | revision ${record.revision} | ${record.definition.name}`, + record.definition.description, + ].filter(Boolean).join(" | ")).join("\n") +} + +export function describeWorkflowDefinition(record: WorkflowDefinitionRecord) { + return [ + `Workflow definition ${record.id}`, + `Name: ${record.definition.name}`, + `Revision: ${record.revision}`, + record.definition.description ? `Description: ${record.definition.description}` : "", + `Canonical definition:\n${record.canonical}`, + ].filter(Boolean).join("\n") +} + +type WorkflowRequester = Pick, "requestJson"> + +export function createWorkflowTools(config: CodeNomadConfig, requester: WorkflowRequester = createCodeNomadRequester(config)) { + const request = (path: string, init?: RequestInit) => requester.requestJson(`/workflow-runs${path}`, init) + const requestDefinition = (path: string, init?: RequestInit) => requester.requestJson(`/workflow-definitions${path}`, init) + + return { + list_codenomad_workflow_definitions: tool({ + description: "List saved workflow definitions available to start at their current revision.", + args: {}, + async execute() { + const response = await requestDefinition<{ definitions: WorkflowDefinitionRecord[] }>("") + return describeWorkflowDefinitions(response.definitions) + }, + }), + get_codenomad_workflow_definition: tool({ + description: "Inspect the current revision of a saved workflow definition. This tool cannot inspect historical revisions.", + args: { definition_id: tool.schema.string().describe("Saved workflow definition ID") }, + async execute(args) { + const record = await requestDefinition(`/${encodeURIComponent(args.definition_id)}`) + return describeWorkflowDefinition(record) + }, + }), + start_codenomad_workflow_definition: tool({ + description: "Start the current revision of a saved workflow definition. Approval and input gates require a human in the CodeNomad UI.", + args: { + definition_id: tool.schema.string().describe("Saved workflow definition ID"), + objective: tool.schema.string().optional().describe("Optional objective; defaults to the definition name"), + inputs_json: tool.schema.string().optional().describe("Optional workflow inputs as a JSON object"), + }, + async execute(args) { + const inputs = parseWorkflowInputs(args.inputs_json) + const run = await requestDefinition(`/${encodeURIComponent(args.definition_id)}/start`, { + method: "POST", + body: JSON.stringify({ + ...(args.objective ? { objective: args.objective } : {}), + ...(inputs ? { inputs } : {}), + }), + }) + return `${summarize(run)}\nStarted the current saved definition revision. Only a human can manage approval, input, pause/resume, and recovery actions in the CodeNomad UI.` + }, + }), + list_codenomad_workflows: tool({ + description: "List workflow runs managed by CodeNomad for this workspace.", + args: {}, + async execute() { + const response = await request<{ runs: WorkflowRun[] }>("") + if (response.runs.length === 0) return "No CodeNomad workflow runs found." + return response.runs.map((run) => run.definitionId || run.executionNodes + ? `Objective: ${run.objective}\n${summarize(run)}` + : `${run.id} | ${run.status} | ${run.objective}`).join("\n\n") + }, + }), + get_codenomad_workflow: tool({ + description: "Inspect one CodeNomad workflow run and its role sessions.", + args: { run_id: tool.schema.string().describe("Workflow run ID") }, + async execute(args) { + return details(await request(`/${encodeURIComponent(args.run_id)}`)) + }, + }), + approve_codenomad_workflow: tool({ + description: + "Show the pending approval details. Only a user in the CodeNomad Workflows panel can approve and continue the workflow.", + args: { run_id: tool.schema.string().describe("Workflow run ID") }, + async execute(args) { + const run = await request(`/${encodeURIComponent(args.run_id)}`) + return `${details(run)}\nApproval was not applied. Ask the user to approve this run in the CodeNomad Workflows panel.` + }, + }), + cancel_codenomad_workflow: tool({ + description: "Cancel a running or review-pending CodeNomad workflow.", + args: { run_id: tool.schema.string().describe("Workflow run ID") }, + async execute(args) { + const run = await request(`/${encodeURIComponent(args.run_id)}/cancel`, { method: "POST" }) + return summarize(run) + }, + }), + } +} diff --git a/packages/opencode-plugin/tsconfig.json b/packages/opencode-plugin/tsconfig.json index 09a866276..a7da85a5f 100644 --- a/packages/opencode-plugin/tsconfig.json +++ b/packages/opencode-plugin/tsconfig.json @@ -13,5 +13,5 @@ "types": ["node"] }, "include": ["plugin/**/*.ts"], - "exclude": ["dist", "node_modules"] + "exclude": ["dist", "node_modules", "plugin/**/*.test.ts"] } diff --git a/packages/server/src/api-types.ts b/packages/server/src/api-types.ts index d61b12f0e..9eed92c87 100644 --- a/packages/server/src/api-types.ts +++ b/packages/server/src/api-types.ts @@ -16,6 +16,8 @@ export type WorkspaceStatus = "starting" | "ready" | "stopped" | "error" export interface WorkspaceDescriptor { id: string + /** Stable across desktop restore; distinct for force-created instances of the same path. */ + lineageId?: string /** Correlates creation events with the client request that initiated them. */ requestId?: string /** Absolute path on the server host. */ @@ -39,6 +41,7 @@ export interface WorkspaceDescriptor { export interface WorkspaceCreateRequest { path: string + lineageId?: string name?: string binaryPath?: string requestId?: string @@ -293,6 +296,314 @@ export interface InstanceStreamEvent { [key: string]: unknown } +export type WorkflowRunStatus = + | "running" + | "pausing" + | "paused" + | "waiting_for_review" + | "waiting_for_input" + | "completed" + | "failed" + | "cancelled" + | "interrupted" + | "recovery_required" +export type WorkflowStepStatus = "pending" | "running" | "completed" | "failed" | "cancelled" + +export interface WorkflowModelSelection { + providerID: string + modelID: string +} + +export interface WorkflowStageConfig { + id: string + title: string + instructions: string + agent?: string + model?: WorkflowModelSelection + requiresApproval?: boolean +} + +export interface WorkflowValueRef { + /** Dot path rooted at inputs, nodes, or vars, for example `nodes.plan.output.steps`. */ + $ref: string +} + +export type WorkflowValue = null | boolean | number | string | WorkflowValueRef | WorkflowValue[] | { + [key: string]: WorkflowValue +} + +export type WorkflowCondition = boolean | { + value: WorkflowValue + equals?: WorkflowValue + notEquals?: WorkflowValue + exists?: boolean + truthy?: boolean +} + +export interface WorkflowRetryPolicy { + maxAttempts: number + delayMs?: number + /** Required when maxAttempts > 1 because an admitted operation may have completed before an error was observed. */ + idempotent?: boolean +} + +export interface WorkflowNodeBase { + id: string + title?: string + if?: WorkflowCondition +} + +export interface WorkflowSequenceNode extends WorkflowNodeBase { + type: "sequence" + steps: WorkflowNode[] +} + +export interface WorkflowParallelNode extends WorkflowNodeBase { + type: "parallel" + branches: WorkflowNode[] + maxConcurrency?: number +} + +export interface WorkflowForeachNode extends WorkflowNodeBase { + type: "foreach" + items: WorkflowValue + item: string + body: WorkflowNode + maxItems: number + maxConcurrency?: number +} + +export interface WorkflowRepeatNode extends WorkflowNodeBase { + type: "repeat" + body: WorkflowNode + maxIterations: number + while?: WorkflowCondition +} + +export interface WorkflowAgentNode extends WorkflowNodeBase { + type: "agent" + instructions: string + context?: WorkflowValue + agent?: string + model?: WorkflowModelSelection + /** Model tool access only. The installed SDK cannot invoke an arbitrary tool deterministically. */ + tools?: string[] + outputSchema?: Record + retry?: WorkflowRetryPolicy + timeoutMs?: number +} + +export interface WorkflowShellNode extends WorkflowNodeBase { + type: "shell" + command: string + agent: string + model?: WorkflowModelSelection + retry?: WorkflowRetryPolicy + timeoutMs?: number +} + +export interface WorkflowGateNode extends WorkflowNodeBase { + type: "gate" + gate: "approval" | "input" + prompt: string + inputSchema?: Record +} + +export interface WorkflowSavedCallNode extends WorkflowNodeBase { + type: "workflow" + definitionId: string + /** Omitted definitions are resolved and pinned to the current revision when the run is admitted. */ + definitionRevision?: number + inputs?: Record +} + +export interface WorkflowBranchNode extends WorkflowNodeBase { + type: "condition" + condition: WorkflowCondition + then: WorkflowNode + else?: WorkflowNode +} + +export type WorkflowNode = + | WorkflowSequenceNode + | WorkflowParallelNode + | WorkflowForeachNode + | WorkflowRepeatNode + | WorkflowAgentNode + | WorkflowShellNode + | WorkflowGateNode + | WorkflowBranchNode + | WorkflowSavedCallNode + +export interface WorkflowBudget { + /** Stops admitting actions once observed cost reaches this value; provider reporting may let the final admitted action overshoot. */ + maxCost?: number + /** Stops admitting actions once observed tokens reach this value; provider reporting may let the final admitted action overshoot. */ + maxTokens?: number +} + +export interface WorkflowDefinitionV1 { + version: 1 + id: string + name: string + description?: string + root: WorkflowNode + budget?: WorkflowBudget + maxConcurrency?: number + maxExpandedNodes?: number +} + +export interface WorkflowDefinitionRecord { + id: string + revision: number + definition: WorkflowDefinitionV1 + canonical: string + createdAt: string + updatedAt: string +} + +export interface WorkflowSavedDefinitionSnapshot { + id: string + revision: number + definition: WorkflowDefinitionV1 +} + +export type WorkflowRunWorktreePolicy = + | { mode: "current" } + | { mode: "existing"; slug: string } + | { mode: "new"; slug: string } + +export interface WorkflowRunWorktreeSelection { + policy: WorkflowRunWorktreePolicy + sourceWorkspaceId: string + sourceWorkspaceLineageId: string + sourceWorkspacePath: string + workspaceId: string + directory: string + slug?: string + branch?: string + created: boolean +} + +export interface WorkflowUsage { + cost: number + tokens: number + inputTokens: number + outputTokens: number + reasoningTokens: number + cacheReadTokens: number + cacheWriteTokens: number +} + +export type WorkflowExecutionNodeStatus = + | "pending" + | "running" + | "waiting" + | "completed" + | "skipped" + | "failed" + | "cancelled" + | "interrupted" + +export interface WorkflowExecutionNode { + id: string + instanceKey: string + /** Identifies one root or saved-definition invocation for nodes. reference isolation. */ + definitionInvocationKey?: string + definitionNodeId: string + type: WorkflowNode["type"] + status: WorkflowExecutionNodeStatus + attempt: number + parentInstanceKey?: string + sessionIds?: string[] + output?: unknown + outputTruncated?: boolean + error?: string + usage?: WorkflowUsage + startedAt?: string + completedAt?: string +} + +export interface WorkflowPendingGate { + executionNodeId: string + definitionNodeId: string + gate: "approval" | "input" + prompt: string + inputSchema?: Record +} + +export interface WorkflowRunStep extends WorkflowStageConfig { + status: WorkflowStepStatus + sessionId?: string + output?: unknown + outputTruncated?: boolean + error?: string + startedAt?: string + completedAt?: string +} + +export interface WorkflowRun { + id: string + workspaceId: string + workspaceLineageId: string + workspacePath: string + initiatorSessionId?: string + objective: string + status: WorkflowRunStatus + rootSessionId?: string + activeStepId?: string + pendingReviewStepId?: string + steps: WorkflowRunStep[] + revision?: number + definitionId?: string + definitionRevision?: number + definitionSnapshot?: WorkflowDefinitionV1 + savedDefinitionSnapshots?: WorkflowSavedDefinitionSnapshot[] + worktreeSelection?: WorkflowRunWorktreeSelection + inputs?: Record + executionNodes?: WorkflowExecutionNode[] + pendingGate?: WorkflowPendingGate + usage?: WorkflowUsage + pauseRequested?: boolean + error?: string + createdAt: string + updatedAt: string +} + +export interface WorkflowRunCreateRequest { + workspaceId: string + initiatorSessionId?: string + objective: string + stages: WorkflowStageConfig[] +} + +export interface WorkflowDefinitionRunCreateRequest { + workspaceId: string + initiatorSessionId?: string + objective?: string + definitionId: string + definitionRevision?: number + inputs?: Record + worktree?: WorkflowRunWorktreePolicy +} + +export type WorkflowRunStartRequest = WorkflowRunCreateRequest | WorkflowDefinitionRunCreateRequest + +export type WorkflowDefinitionPayload = + | { source: string; definition?: never } + | { definition: WorkflowDefinitionV1; source?: never } + +export type WorkflowDefinitionUpdateRequest = WorkflowDefinitionPayload & { expectedRevision: number } + +export interface WorkflowGateAnswerRequest { + executionNodeId: string + answer: unknown +} + +export interface WorkflowResumeRequest { + confirmRecovery?: boolean +} + export type SideCarKind = "port" export type SideCarPrefixMode = "strip" | "preserve" diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 0f06a2cf0..a31fcb3e7 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -36,6 +36,7 @@ import { createServerShutdownHandler, orchestrateServerShutdown, type ServerShut import { AutoAcceptManager } from "./permissions/auto-accept-manager" import { createOpencodePermissionReplier } from "./permissions/opencode-replier" import { createOpencodeYoloPersistence } from "./permissions/opencode-yolo-metadata" +import { WorkflowManager } from "./workflows/manager" const require = createRequire(import.meta.url) @@ -375,6 +376,13 @@ async function main() { getServerBaseUrl: () => serverMeta.localUrl, nodeExtraCaCertsPath, }) + const workflowManager = new WorkflowManager({ + workspaceManager, + eventBus, + storageDir: path.join(configDir, "workflow-runs"), + definitionsDir: path.join(configDir, "workflow-definitions"), + logger: logger.child({ component: "workflows" }), + }) const fileSystemBrowser = new FileSystemBrowser({ rootDir: options.rootDir, unrestricted: options.unrestrictedRoot, @@ -499,6 +507,7 @@ async function main() { remoteProxySessionManager, yoloManager, sessionMetadataPersistence, + workflowManager, uiStaticDir: uiResolution.uiStaticDir ?? DEFAULT_UI_STATIC_DIR, uiDevServerUrl: uiResolution.uiDevServerUrl, logger, @@ -528,6 +537,7 @@ async function main() { remoteProxySessionManager, yoloManager, sessionMetadataPersistence, + workflowManager, uiStaticDir: uiResolution.uiStaticDir ?? DEFAULT_UI_STATIC_DIR, uiDevServerUrl: undefined, logger, @@ -623,6 +633,7 @@ async function main() { orchestrateServerShutdown( { stopInstanceEventBridge: () => instanceEventBridge.shutdown(), + stopWorkflowRuns: () => workflowManager.shutdown(), stopSidecars: () => sidecarManager.shutdown(), stopClientConnections: () => clientConnectionManager.shutdown(), stopRemoteProxySessions: () => remoteProxySessionManager.shutdown(), diff --git a/packages/server/src/server/http-server.test.ts b/packages/server/src/server/http-server.test.ts new file mode 100644 index 000000000..da221c518 --- /dev/null +++ b/packages/server/src/server/http-server.test.ts @@ -0,0 +1,141 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import pino from "pino" + +import { EventBus } from "../events/bus" +import { createHttpServer } from "./http-server" + +test("browser origins and plugin callbacks use separate security gates", async () => { + const logger = pino({ level: "silent" }) + const workspace = { + id: "workspace", + path: process.cwd(), + status: "ready", + proxyPath: "/workspaces/workspace/instance", + binaryId: "opencode", + binaryLabel: "opencode", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + let deletionGuardWired = false + const workspaceManager = { + get: (id: string) => id === workspace.id ? workspace : undefined, + list: () => [workspace], + getPluginCallbackAuthorizationHeader: (id: string) => id === workspace.id ? "Bearer callback-secret" : undefined, + getInstanceAuthorizationHeader: () => "Basic shared-opencode-secret", + setDeletionGuard: () => { deletionGuardWired = true }, + } + const authManager = { + isTokenBootstrapEnabled: () => false, + isLoopbackRequest: () => true, + getSessionFromRequest: (request: { headers: { cookie?: string } }) => request.headers.cookie === "session=valid" ? { id: "session" } : null, + } + const server = createHttpServer({ + bindHost: "0.0.0.0", + bindPort: 0, + defaultPort: 0, + protocol: "http", + workspaceManager, + settings: {}, + fileSystemBrowser: {}, + eventBus: new EventBus(), + serverMeta: { + localUrl: "http://localhost:4000", + remoteUrl: "http://192.168.1.2:4000", + addresses: [], + }, + instanceStore: {}, + speechService: {}, + sidecarManager: {}, + previewManager: {}, + authManager, + clientConnectionManager: { pong: () => true, register: () => () => undefined }, + pluginChannel: {}, + voiceModeManager: {}, + remoteProxySessionManager: {}, + yoloManager: {}, + sessionMetadataPersistence: {}, + workflowManager: { list: async () => [] }, + uiStaticDir: "", + uiDevServerUrl: "http://localhost:3000", + logger, + } as never) + assert.equal(deletionGuardWired, true) + await server.instance.ready() + try { + const hostile = await server.instance.inject({ + method: "POST", + url: "/api/client-connections/pong", + headers: { cookie: "session=valid", origin: "https://attacker.example" }, + payload: { clientId: "client", connectionId: "connection" }, + }) + assert.equal(hostile.statusCode, 403) + assert.equal(hostile.headers["access-control-allow-origin"], undefined) + + const trusted = await server.instance.inject({ + method: "POST", + url: "/api/client-connections/pong", + headers: { cookie: "session=valid", origin: "http://localhost:3000" }, + payload: { clientId: "client", connectionId: "connection" }, + }) + assert.equal(trusted.statusCode, 204) + assert.equal(trusted.headers["access-control-allow-origin"], "http://localhost:3000") + + const alias = await server.instance.inject({ + method: "POST", + url: "/api/client-connections/pong", + headers: { cookie: "session=valid", host: "codenomad.local", origin: "http://codenomad.local" }, + payload: { clientId: "client", connectionId: "connection" }, + }) + assert.equal(alias.statusCode, 204) + + for (const headers of [ + {}, + { cookie: "session=valid" }, + { authorization: "Basic shared-opencode-secret" }, + ]) { + const rejected = await server.instance.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + headers, + payload: { type: "test.event" }, + }) + assert.equal(rejected.statusCode, 401) + } + + const callback = await server.instance.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + headers: { authorization: "Bearer callback-secret" }, + payload: { type: "test.event" }, + }) + assert.equal(callback.statusCode, 204) + + const noncanonicalCallback = await server.instance.inject({ + method: "POST", + url: "/workspaces/workspace/plugin//event", + headers: { cookie: "session=valid" }, + payload: { type: "test.event" }, + }) + assert.equal(noncanonicalCallback.statusCode, 404) + + const trustedSsePending = server.instance.inject({ + method: "GET", + url: "/api/events?clientId=trusted&connectionId=trusted", + headers: { cookie: "session=valid", origin: "http://localhost:3000" }, + }) + const hostileSsePending = server.instance.inject({ + method: "GET", + url: "/api/events?clientId=hostile&connectionId=hostile", + headers: { cookie: "session=valid", origin: "https://attacker.example" }, + }) + await new Promise((resolve) => setImmediate(resolve)) + await server.stop() + const [trustedSse, hostileSse] = await Promise.all([trustedSsePending, hostileSsePending]) + assert.equal(trustedSse.headers["access-control-allow-origin"], "http://localhost:3000") + assert.equal(trustedSse.headers["access-control-allow-credentials"], "true") + assert.equal(hostileSse.headers["access-control-allow-origin"], undefined) + } finally { + if (server.instance.server.listening) await server.stop() + } +}) diff --git a/packages/server/src/server/http-server.ts b/packages/server/src/server/http-server.ts index 8df2832be..b0383b4af 100644 --- a/packages/server/src/server/http-server.ts +++ b/packages/server/src/server/http-server.ts @@ -8,7 +8,7 @@ import path from "path" import { connect as connectTls, type TLSSocket } from "tls" import { fetch, type Headers } from "undici" import type { Logger } from "../logger" -import { WorkspaceManager } from "../workspaces/manager" +import { WorkspaceDeletionBlockedError, WorkspaceManager } from "../workspaces/manager" import type { SettingsService } from "../settings/service" import { FileSystemBrowser } from "../filesystem/browser" @@ -47,6 +47,8 @@ import type { SideCarManager } from "../sidecars/manager" import type { PreviewManager } from "../previews/manager" import type { RemoteProxySessionManager } from "./remote-proxy" import { createOpenCodeUpdateService } from "../opencode-update/service" +import type { WorkflowManager } from "../workflows/manager" +import { registerWorkflowRoutes } from "./routes/workflows" interface HttpServerDeps { bindHost: string @@ -71,6 +73,7 @@ interface HttpServerDeps { remoteProxySessionManager: RemoteProxySessionManager yoloManager: AutoAcceptManager sessionMetadataPersistence: OpencodeYoloPersistence + workflowManager: WorkflowManager uiStaticDir: string uiDevServerUrl?: string logger: Logger @@ -83,6 +86,16 @@ interface HttpServerStartResult { } export function createHttpServer(deps: HttpServerDeps) { + deps.workspaceManager.setDeletionGuard?.((workspace, operation) => + deps.workflowManager.withWorkspaceOwnershipLease({ + id: workspace.id, + lineageId: workspace.lineageId, + path: workspace.path, + }, async (owned) => { + if (owned) throw new WorkspaceDeletionBlockedError(workspace.id) + return operation() + })) + // Fastify's type-level RawServer inference gets noisy when toggling HTTP vs HTTPS. // We keep the runtime behavior correct and cast the instance to a generic FastifyInstance. const app = Fastify( @@ -130,12 +143,9 @@ export function createHttpServer(deps: HttpServerDeps) { done() }) - const allowedDevOrigins = new Set(["http://localhost:3000", "http://127.0.0.1:3000"]) - const isLoopbackHost = (host: string) => host === "127.0.0.1" || host === "::1" || host.startsWith("127.") - - const getSelfOrigins = (): Set => { + const getAllowedOrigins = (): Set => { const origins = new Set() - const candidates: Array = [deps.serverMeta.localUrl, deps.serverMeta.remoteUrl] + const candidates: Array = [deps.serverMeta.localUrl, deps.serverMeta.remoteUrl, deps.uiDevServerUrl] for (const candidate of candidates) { if (!candidate) continue try { @@ -161,24 +171,11 @@ export function createHttpServer(deps: HttpServerDeps) { return } - const selfOrigins = getSelfOrigins() - if (selfOrigins.has(origin)) { + if (getAllowedOrigins().has(origin)) { cb(null, true) return } - if (allowedDevOrigins.has(origin)) { - cb(null, true) - return - } - - // When we bind to a non-loopback host (e.g., 0.0.0.0 or LAN IP), allow cross-origin UI access. - if (deps.bindHost === "0.0.0.0" || !isLoopbackHost(deps.bindHost)) { - cb(null, true) - return - } - - cb(null, false) }, credentials: true, @@ -205,6 +202,25 @@ export function createHttpServer(deps: HttpServerDeps) { app.addHook("preHandler", (request, reply, done) => { const rawUrl = request.raw.url ?? request.url const pathname = (rawUrl.split("?")[0] ?? "").trim() + const session = deps.authManager.getSessionFromRequest(request) + const originHeader = request.headers.origin + const origin = Array.isArray(originHeader) ? originHeader[0] : originHeader + const fetchSiteHeader = request.headers["sec-fetch-site"] + const fetchSite = Array.isArray(fetchSiteHeader) ? fetchSiteHeader[0] : fetchSiteHeader + let requestOrigin: string | undefined + try { + if (request.headers.host) requestOrigin = new URL(`${request.protocol}://${request.headers.host}`).origin + } catch { + // Invalid Host values cannot establish a trusted browser origin. + } + if ( + session + && ["POST", "PUT", "PATCH", "DELETE"].includes(request.method) + && ((origin && origin !== requestOrigin && !getAllowedOrigins().has(origin)) || fetchSite === "cross-site") + ) { + reply.code(403).send({ error: "Cross-origin mutation forbidden" }) + return + } const publicApiPaths = new Set(["/api/auth/login", "/api/auth/token", "/api/auth/status", "/api/auth/logout"]) const publicPagePaths = new Set(["/login"]) @@ -222,23 +238,29 @@ export function createHttpServer(deps: HttpServerDeps) { return } - const session = deps.authManager.getSessionFromRequest(request) + const pluginMatch = pathname.match(/^\/workspaces\/([^/]+)\/plugin(?:\/|$)/) + const provided = Array.isArray(request.headers.authorization) + ? request.headers.authorization[0] + : request.headers.authorization + let pluginAuthorized = false + if (pluginMatch) { + const expected = deps.workspaceManager.getPluginCallbackAuthorizationHeader(pluginMatch[1] ?? "") + pluginAuthorized = Boolean(expected && provided && provided === expected) + } + const requiresPluginCapability = Boolean(pluginMatch) && ( + (request.method === "GET" && pathname.endsWith("/plugin/events")) + || (request.method === "POST" && pathname.endsWith("/plugin/event")) + ) + if (requiresPluginCapability && !pluginAuthorized) { + sendUnauthorized(request, reply) + return + } const requiresAuthForApi = pathname.startsWith("/api/") || pathname.startsWith("/workspaces/") || pathname.startsWith("/sidecars/") || pathname.startsWith("/previews/") if (requiresAuthForApi && !session) { - // Allow OpenCode plugin -> CodeNomad calls with per-instance basic auth. - const pluginMatch = pathname.match(/^\/workspaces\/([^/]+)\/plugin(?:\/|$)/) - if (pluginMatch) { - const workspaceId = pluginMatch[1] - const expected = deps.workspaceManager.getInstanceAuthorizationHeader(workspaceId) - const provided = Array.isArray(request.headers.authorization) - ? request.headers.authorization[0] - : request.headers.authorization - - if (expected && provided && provided === expected) { - done() - return - } + if (pluginAuthorized) { + done() + return } sendUnauthorized(request, reply) @@ -293,6 +315,7 @@ export function createHttpServer(deps: HttpServerDeps) { registerWorktreeRoutes(app, { workspaceManager: deps.workspaceManager, sessionMetadataPersistence: deps.sessionMetadataPersistence, + workflowManager: deps.workflowManager, }) registerStorageRoutes(app, { instanceStore: deps.instanceStore, @@ -326,6 +349,7 @@ export function createHttpServer(deps: HttpServerDeps) { }) registerBackgroundProcessRoutes(app, { backgroundProcessManager }) registerYoloRoutes(app, { yoloManager: deps.yoloManager }) + registerWorkflowRoutes(app, { workflowManager: deps.workflowManager }) registerInstanceProxyRoutes(app, { workspaceManager: deps.workspaceManager, logger: proxyLogger }) diff --git a/packages/server/src/server/routes/events.test.ts b/packages/server/src/server/routes/events.test.ts new file mode 100644 index 000000000..516e9efe5 --- /dev/null +++ b/packages/server/src/server/routes/events.test.ts @@ -0,0 +1,29 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import Fastify from "fastify" + +import { EventBus } from "../../events/bus" +import { registerEventRoutes } from "./events" + +test("SSE does not reflect request origins", async () => { + const app = Fastify({ logger: false }) + registerEventRoutes(app, { + eventBus: new EventBus(), + registerClient: (close) => { + setImmediate(close) + return () => undefined + }, + logger: { debug: () => undefined, isLevelEnabled: () => false } as never, + connectionManager: { register: () => () => undefined } as never, + }) + + const response = await app.inject({ + method: "GET", + url: "/api/events?clientId=client&connectionId=connection", + headers: { origin: "https://attacker.example" }, + }) + + assert.equal(response.headers["access-control-allow-origin"], undefined) + assert.equal(response.headers["access-control-allow-credentials"], undefined) + await app.close() +}) diff --git a/packages/server/src/server/routes/events.ts b/packages/server/src/server/routes/events.ts index 158266e12..2aee15526 100644 --- a/packages/server/src/server/routes/events.ts +++ b/packages/server/src/server/routes/events.ts @@ -1,4 +1,4 @@ -import { FastifyInstance } from "fastify" +import { FastifyInstance, type FastifyReply } from "fastify" import { z } from "zod" import { EventBus } from "../../events/bus" import { WorkspaceEventPayload } from "../../api-types" @@ -29,12 +29,10 @@ export function registerEventRoutes(app: FastifyInstance, deps: RouteDeps) { const connection = ConnectionQuerySchema.parse(request.query ?? {}) deps.logger.debug({ clientId }, "SSE client connected") - const origin = request.headers.origin ?? "*" - reply.raw.setHeader("Access-Control-Allow-Origin", origin) - reply.raw.setHeader("Access-Control-Allow-Credentials", "true") - reply.raw.setHeader("Content-Type", "text/event-stream") - reply.raw.setHeader("Cache-Control", "no-cache") - reply.raw.setHeader("Connection", "keep-alive") + reply.header("Content-Type", "text/event-stream") + reply.header("Cache-Control", "no-cache") + reply.header("Connection", "keep-alive") + copyReplyHeadersToRaw(reply) reply.raw.flushHeaders?.() reply.hijack() @@ -87,3 +85,9 @@ export function registerEventRoutes(app: FastifyInstance, deps: RouteDeps) { reply.code(204).send() }) } + +function copyReplyHeadersToRaw(reply: FastifyReply): void { + for (const [name, value] of Object.entries(reply.getHeaders())) { + if (value !== undefined) reply.raw.setHeader(name, value) + } +} diff --git a/packages/server/src/server/routes/plugin.test.ts b/packages/server/src/server/routes/plugin.test.ts new file mode 100644 index 000000000..684d326b8 --- /dev/null +++ b/packages/server/src/server/routes/plugin.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import Fastify from "fastify" + +import { EventBus } from "../../events/bus" +import { registerPluginRoutes } from "./plugin" + +test("plugin event callbacks require the capability on the canonical handler", async () => { + const app = Fastify({ logger: false }) + registerPluginRoutes(app, { + workspaceManager: { + get: (id: string) => id === "workspace" ? { id, status: "ready" } : undefined, + getPluginCallbackAuthorizationHeader: (id: string) => id === "workspace" ? "Bearer callback-secret" : undefined, + } as never, + eventBus: new EventBus(), + logger: { debug: () => undefined } as never, + channel: {} as never, + voiceModeManager: {} as never, + }) + + const missing = await app.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + payload: { type: "test.event" }, + }) + const authorized = await app.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + headers: { authorization: "Bearer callback-secret" }, + payload: { type: "test.event" }, + }) + const doubledSlash = await app.inject({ + method: "POST", + url: "/workspaces/workspace/plugin//event", + headers: { authorization: "Bearer callback-secret" }, + payload: { type: "test.event" }, + }) + + assert.equal(missing.statusCode, 401) + assert.equal(authorized.statusCode, 204) + assert.equal(doubledSlash.statusCode, 404) + await app.close() +}) diff --git a/packages/server/src/server/routes/plugin.ts b/packages/server/src/server/routes/plugin.ts index aef570072..c8eca6ad9 100644 --- a/packages/server/src/server/routes/plugin.ts +++ b/packages/server/src/server/routes/plugin.ts @@ -1,4 +1,4 @@ -import { FastifyInstance } from "fastify" +import { FastifyInstance, type FastifyReply, type FastifyRequest } from "fastify" import { z } from "zod" import type { VoiceModeStateResponse } from "../../api-types" import type { WorkspaceManager } from "../../workspaces/manager" @@ -7,6 +7,7 @@ import type { Logger } from "../../logger" import { PluginChannelManager } from "../../plugins/channel" import { buildPingEvent, handlePluginEvent } from "../../plugins/handlers" import { VoiceModeManager } from "../../plugins/voice-mode" +import { sendUnauthorized } from "../../auth/http-auth" interface RouteDeps { workspaceManager: WorkspaceManager @@ -29,15 +30,24 @@ const VoiceModeStateSchema = z.object({ export function registerPluginRoutes(app: FastifyInstance, deps: RouteDeps) { app.get<{ Params: { id: string } }>("/workspaces/:id/plugin/events", (request, reply) => { + if (!isCanonicalPluginPath(request, request.params.id, "events")) { + reply.code(404).send({ error: "Unknown plugin endpoint" }) + return + } + if (!hasPluginCapability(request, request.params.id, deps)) { + sendUnauthorized(request, reply) + return + } const workspace = deps.workspaceManager.get(request.params.id) if (!workspace) { reply.code(404).send({ error: "Workspace not found" }) return } - reply.raw.setHeader("Content-Type", "text/event-stream") - reply.raw.setHeader("Cache-Control", "no-cache") - reply.raw.setHeader("Connection", "keep-alive") + reply.header("Content-Type", "text/event-stream") + reply.header("Cache-Control", "no-cache") + reply.header("Connection", "keep-alive") + copyReplyHeadersToRaw(reply) reply.raw.flushHeaders?.() reply.hijack() @@ -80,27 +90,46 @@ export function registerPluginRoutes(app: FastifyInstance, deps: RouteDeps) { return { enabled: payload.enabled } }) - const handleWildcard = async (request: any, reply: any) => { - const workspaceId = request.params.id as string + app.post<{ Params: { id: string } }>("/workspaces/:id/plugin/event", async (request, reply) => { + const workspaceId = request.params.id + if (!isCanonicalPluginPath(request, workspaceId, "event")) { + reply.code(404).send({ error: "Unknown plugin endpoint" }) + return + } + if (!hasPluginCapability(request, workspaceId, deps)) { + sendUnauthorized(request, reply) + return + } const workspace = deps.workspaceManager.get(workspaceId) if (!workspace) { reply.code(404).send({ error: "Workspace not found" }) return } - const suffix = (request.params["*"] as string | undefined) ?? "" - const normalized = suffix.replace(/^\/+/, "") + const parsed = PluginEventSchema.parse(request.body ?? {}) + handlePluginEvent(workspaceId, parsed, { workspaceManager: deps.workspaceManager, eventBus: deps.eventBus, logger: deps.logger }) + reply.code(204).send() + }) - if (normalized === "event" && request.method === "POST") { - const parsed = PluginEventSchema.parse(request.body ?? {}) - handlePluginEvent(workspaceId, parsed, { workspaceManager: deps.workspaceManager, eventBus: deps.eventBus, logger: deps.logger }) - reply.code(204).send() - return - } + app.all("/workspaces/:id/plugin/*", (_request, reply) => reply.code(404).send({ error: "Unknown plugin endpoint" })) + app.all("/workspaces/:id/plugin", (_request, reply) => reply.code(404).send({ error: "Unknown plugin endpoint" })) +} - reply.code(404).send({ error: "Unknown plugin endpoint" }) - } +function isCanonicalPluginPath(request: FastifyRequest, workspaceId: string, endpoint: string): boolean { + const pathname = (request.raw.url ?? request.url).split("?")[0] + return pathname === `/workspaces/${encodeURIComponent(workspaceId)}/plugin/${endpoint}` +} + +function hasPluginCapability(request: FastifyRequest, workspaceId: string, deps: RouteDeps): boolean { + const provided = Array.isArray(request.headers.authorization) + ? request.headers.authorization[0] + : request.headers.authorization + const expected = deps.workspaceManager.getPluginCallbackAuthorizationHeader(workspaceId) + return Boolean(expected && provided === expected) +} - app.all("/workspaces/:id/plugin/*", handleWildcard) - app.all("/workspaces/:id/plugin", handleWildcard) +function copyReplyHeadersToRaw(reply: FastifyReply): void { + for (const [name, value] of Object.entries(reply.getHeaders())) { + if (value !== undefined) reply.raw.setHeader(name, value) + } } diff --git a/packages/server/src/server/routes/workflows.test.ts b/packages/server/src/server/routes/workflows.test.ts new file mode 100644 index 000000000..7fad2a4f1 --- /dev/null +++ b/packages/server/src/server/routes/workflows.test.ts @@ -0,0 +1,281 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import Fastify from "fastify" +import { WORKFLOW_DEFINITION_REVISION_LIMIT } from "../../workflows/definition-schema" +import { WorkflowRunError, type WorkflowManager } from "../../workflows/manager" +import { registerWorkflowRoutes } from "./workflows" + +describe("workflow routes", () => { + it("rejects generic plugin starts and scopes plugin requests to their workspace", async () => { + const calls: unknown[] = [] + const workflowManager = { + start: async (input: unknown) => { + calls.push(input) + return { id: "00000000-0000-4000-8000-000000000001", workspaceId: "workspace-a", status: "running" } + }, + list: async () => [], + get: async () => ({ id: "run", workspaceId: "workspace-b" }), + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + const rejected = await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs", + payload: { + objective: "Ship it", + stages: [{ id: "build", title: "Build", instructions: "Implement" }], + }, + }) + assert.equal(rejected.statusCode, 404) + assert.equal(calls.length, 0) + + const foreign = await app.inject({ + method: "GET", + url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001", + }) + assert.equal(foreign.statusCode, 404) + + const pluginApproval = await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001/approve", + }) + assert.equal(pluginApproval.statusCode, 404) + await app.close() + }) + + it("exposes definition CRUD to host routes and only list/get/start to a workspace plugin", async () => { + const calls: Array<[string, ...unknown[]]> = [] + const record = { id: "deploy", revision: 2, definition: { id: "deploy", name: "Deploy" } } + const workflowManager = { + listDefinitions: async () => [record], + getDefinition: async (id: string, revision?: number) => { calls.push(["get", id, revision]); return record }, + createDefinition: async (source: unknown) => { calls.push(["create", source]); return record }, + updateDefinition: async (id: string, revision: number, source: unknown) => { calls.push(["update", id, revision, source]); return record }, + deleteDefinition: async (id: string, revision: number) => { calls.push(["delete", id, revision]); return true }, + validateDefinition: () => ({ valid: true }), + start: async (input: unknown) => { calls.push(["start", input]); return { id: "run", workspaceId: "workspace-a" } }, + startLatest: async (input: unknown) => { calls.push(["startLatest", input]); return { id: "run", workspaceId: "workspace-a" } }, + pause: async (id: string) => { calls.push(["pause", id]); return { id } }, + resume: async (id: string, confirm: boolean) => { calls.push(["resume", id, confirm]); return { id } }, + answer: async (id: string, executionNodeId: string, answer: unknown) => { calls.push(["answer", id, executionNodeId, answer]); return { id } }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + assert.equal((await app.inject({ method: "GET", url: "/workspaces/workspace-a/plugin/workflow-definitions" })).statusCode, 200) + const started = await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { objective: "Release", inputs: { environment: "test" } }, + }) + assert.equal(started.statusCode, 202) + assert.deepEqual(calls.at(-1), ["start", { + workspaceId: "workspace-a", definitionId: "deploy", objective: "Release", + inputs: { environment: "test" }, + }]) + assert.equal((await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs", + payload: { + initiatorSessionId: "parent-session", + objective: "Ship it", + stages: [{ id: "build", title: "Build", instructions: "Implement" }], + }, + })).statusCode, 404) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { worktree: { mode: "existing", slug: "review" } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { initiatorSessionId: "parent-session" }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace-a", worktree: { mode: "current", slug: "not-allowed" } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { definitionRevision: 1 }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-runs", + payload: { definitionId: "deploy" }, + })).statusCode, 404) + assert.equal((await app.inject({ + method: "PUT", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy", + payload: { expectedRevision: 2, definition: {} }, + })).statusCode, 404) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001/answer", + payload: { answer: true }, + })).statusCode, 404) + + assert.equal((await app.inject({ + method: "PUT", url: "/api/workflow-definitions/deploy", + payload: { expectedRevision: 2, definition: { version: 1 } }, + })).statusCode, 200) + assert.deepEqual(calls.at(-1), ["update", "deploy", 2, { version: 1 }]) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace-a", definitionRevision: 2, inputs: { environment: "prod" } }, + })).statusCode, 202) + assert.deepEqual(calls.at(-1), ["startLatest", { + workspaceId: "workspace-a", definitionId: "deploy", definitionRevision: 2, inputs: { environment: "prod" }, + }]) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs/00000000-0000-4000-8000-000000000001/answer", + payload: { executionNodeId: "00000000-0000-4000-8000-000000000002", answer: { approved: true } }, + })).statusCode, 200) + assert.deepEqual(calls.at(-1), ["answer", "00000000-0000-4000-8000-000000000001", "00000000-0000-4000-8000-000000000002", { approved: true }]) + await app.close() + }) + + it("keeps legacy and definition starts exclusive and enforces atomic latest revisions", async () => { + const calls: Array<[string, unknown]> = [] + const workflowManager = { + getDefinition: async () => ({ id: "deploy", revision: 2, definition: { id: "deploy" } }), + start: async (input: unknown) => { calls.push(["start", input]); return { id: "legacy" } }, + startLatest: async (input: { definitionRevision?: number }) => { + if (input.definitionRevision !== undefined && input.definitionRevision !== 2) { + throw new WorkflowRunError("Workflow definition revision is stale", 409) + } + calls.push(["startLatest", input]) + return { id: "saved" } + }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + const legacy = { + workspaceId: "workspace", objective: "Ship", + stages: [{ id: "build", title: "Build", instructions: "Build it" }], + } + assert.equal((await app.inject({ method: "POST", url: "/api/workflow-runs", payload: legacy })).statusCode, 202) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs", payload: { ...legacy, definitionId: "deploy" }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs", payload: { + ...legacy, stages: [{ ...legacy.stages[0], definitionId: "deploy" }], + }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs", payload: { + workspaceId: "workspace", definitionId: "deploy", stages: legacy.stages, + }, + })).statusCode, 400) + + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: 1 }, + })).statusCode, 409) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: 2 }, + })).statusCode, 202) + assert.deepEqual(calls.at(-1), ["startLatest", { + workspaceId: "workspace", definitionId: "deploy", definitionRevision: 2, + }]) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: WORKFLOW_DEFINITION_REVISION_LIMIT + 1 }, + })).statusCode, 400) + await app.close() + }) + + it("fails closed when atomic latest start manager integration is unavailable", async () => { + const workflowManager = { + getDefinition: async () => ({ id: "deploy", revision: 2, definition: { id: "deploy" } }), + start: async () => { throw new Error("non-atomic start must not be called") }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + const response = await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: 2 }, + }) + assert.equal(response.statusCode, 501) + await app.close() + }) + + it("bounds definition inputs and gate answers without recursive validation", async () => { + const calls: Array<[string, ...unknown[]]> = [] + const workflowManager = { + start: async (input: unknown) => { calls.push(["start", input]); return { id: "run" } }, + answer: async (...args: unknown[]) => { calls.push(["answer", ...args]); return { id: "run" } }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + const startUrl = "/api/workflow-definitions/deploy/start" + const pluginStartUrl = "/workspaces/workspace/plugin/workflow-definitions/deploy/start" + const answerUrl = "/api/workflow-runs/00000000-0000-4000-8000-000000000001/answer" + const executionNodeId = "00000000-0000-4000-8000-000000000002" + + let nested: unknown = true + for (let depth = 0; depth < 21; depth++) nested = { nested } + assert.equal((await app.inject({ + method: "POST", url: pluginStartUrl, payload: { inputs: nested }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: startUrl, + payload: { workspaceId: "workspace", inputs: { values: Array(50_001).fill(null) } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: startUrl, + payload: { workspaceId: "workspace", inputs: { value: "é".repeat(128_001) } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: answerUrl, payload: { executionNodeId, answer: nested }, + })).statusCode, 400) + assert.equal(calls.length, 0) + + assert.equal((await app.inject({ + method: "POST", url: answerUrl, payload: { executionNodeId, answer: { approved: true } }, + })).statusCode, 200) + assert.deepEqual(calls, [["answer", "00000000-0000-4000-8000-000000000001", executionNodeId, { approved: true }]]) + await app.close() + }) + + it("requires the expected legacy stage when approving", async () => { + const calls: string[] = [] + const workflowManager = { + approve: async (id: string, expectedStepId: string) => { + if (expectedStepId !== "review-stage") throw new WorkflowRunError("Workflow approval is stale", 409) + calls.push(id) + return { id } + }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + const url = "/api/workflow-runs/00000000-0000-4000-8000-000000000001/approve" + + assert.equal((await app.inject({ method: "POST", url })).statusCode, 400) + assert.equal((await app.inject({ method: "POST", url, payload: { expectedStepId: "stale-stage" } })).statusCode, 409) + assert.equal(calls.length, 0) + assert.equal((await app.inject({ method: "POST", url, payload: { expectedStepId: "review-stage" } })).statusCode, 200) + assert.deepEqual(calls, ["00000000-0000-4000-8000-000000000001"]) + await app.close() + }) + + it("cancels plugin-owned runs without joining get", async () => { + const calls: unknown[][] = [] + const workflowManager = { + get: async () => { throw new Error("get must not be called") }, + cancelOwned: async (...args: unknown[]) => { + calls.push(args) + return { id: args[0], workspaceId: args[1], status: "cancelled" } + }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + const response = await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001/cancel", + }) + assert.equal(response.statusCode, 200) + assert.deepEqual(calls, [["00000000-0000-4000-8000-000000000001", "workspace-a"]]) + await app.close() + }) +}) diff --git a/packages/server/src/server/routes/workflows.ts b/packages/server/src/server/routes/workflows.ts new file mode 100644 index 000000000..867a3c589 --- /dev/null +++ b/packages/server/src/server/routes/workflows.ts @@ -0,0 +1,492 @@ +import type { FastifyInstance } from "fastify" +import { z } from "zod" +import type { WorkflowDefinitionRunCreateRequest, WorkflowRun } from "../../api-types" +import { WORKFLOW_DEFINITION_REVISION_LIMIT, WORKFLOW_LIMITS } from "../../workflows/definition-schema" +import { WorkflowDefinitionStoreError } from "../../workflows/definition-store" +import type { WorkflowManager } from "../../workflows/manager" +import { WorkflowRunError } from "../../workflows/manager" + +type WorkflowManagerWithLatestStart = WorkflowManager & { + startLatest?: (input: WorkflowDefinitionRunCreateRequest) => Promise +} + +interface RouteDeps { + workflowManager: WorkflowManagerWithLatestStart +} + +const DefinitionRevisionSchema = z.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT) + +const ModelSchema = z.object({ + providerID: z.string().trim().min(1).max(200), + modelID: z.string().trim().min(1).max(200), +}).strict() + +const StageSchema = z.object({ + id: z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + title: z.string().trim().min(1).max(200), + instructions: z.string().trim().min(1).max(20_000), + agent: z.string().trim().min(1).max(200).optional(), + model: ModelSchema.optional(), + requiresApproval: z.boolean().optional(), +}).strict() + +const CreateObjectSchema = z.object({ + workspaceId: z.string().trim().min(1).max(200), + initiatorSessionId: z.string().trim().min(1).max(200).optional(), + objective: z.string().trim().min(1).max(50_000), + stages: z.array(StageSchema).min(1).max(12), +}).strict() + +const WorktreePolicySchema = z.discriminatedUnion("mode", [ + z.object({ mode: z.literal("current") }).strict(), + z.object({ mode: z.literal("existing"), slug: z.string().trim().min(1).max(200) }).strict(), + z.object({ mode: z.literal("new"), slug: z.string().trim().min(1).max(200) }).strict(), +]) + +const DefinitionStartObjectSchema = z.object({ + workspaceId: z.string().trim().min(1).max(200), + initiatorSessionId: z.string().trim().min(1).max(200).optional(), + objective: z.string().trim().min(1).max(50_000).optional(), + definitionId: z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + definitionRevision: DefinitionRevisionSchema.optional(), + inputs: z.record(z.unknown()).superRefine(validateBoundedJson).optional(), + worktree: WorktreePolicySchema.optional(), +}).strict() + +const requireUniqueStageIds = (value: { stages: Array<{ id: string }> }, ctx: z.RefinementCtx) => { + const ids = new Set() + value.stages.forEach((stage, index) => { + if (ids.has(stage.id)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Stage IDs must be unique", path: ["stages", index, "id"] }) + } + ids.add(stage.id) + }) +} + +const LegacyCreateSchema = CreateObjectSchema.superRefine(requireUniqueStageIds) +const CreateSchema = z.union([LegacyCreateSchema, DefinitionStartObjectSchema]) + +const RunIdSchema = z.string().uuid() +const ListSchema = z.object({ workspaceId: z.string().trim().min(1).max(200).optional() }) +const PluginDefinitionStartSchema = z.object({ + objective: z.string().trim().min(1).max(50_000).optional(), + inputs: z.record(z.unknown()).superRefine(validateBoundedJson).optional(), +}).strict() +const DefinitionIdSchema = z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/) +const DefinitionRevisionQuerySchema = z.object({ + revision: z.coerce.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT).optional(), +}) +const DefinitionSourceSchema = z.object({ + source: z.string().optional(), + definition: z.unknown().optional(), +}).strict().superRefine((value, ctx) => { + const supplied = Number(value.source !== undefined) + Number(Object.prototype.hasOwnProperty.call(value, "definition")) + if (supplied !== 1) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Supply exactly one of source or definition" }) +}) +const DefinitionUpdateSchema = z.object({ + expectedRevision: DefinitionRevisionSchema, + source: z.string().optional(), + definition: z.unknown().optional(), +}).strict().superRefine((value, ctx) => { + const supplied = Number(value.source !== undefined) + Number(Object.prototype.hasOwnProperty.call(value, "definition")) + if (supplied !== 1) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Supply exactly one of source or definition" }) +}) +const DeleteDefinitionQuerySchema = z.object({ + expectedRevision: z.coerce.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT), +}) +const ResumeSchema = z.object({ confirmRecovery: z.boolean().optional() }).strict() +const ApprovalSchema = z.object({ expectedStepId: z.string().trim().min(1).max(100) }).strict() +const AnswerSchema = z.object({ executionNodeId: z.string().uuid(), answer: z.unknown() }).strict().superRefine((value, ctx) => { + if (!Object.prototype.hasOwnProperty.call(value, "answer")) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "answer is required" }) + else validateBoundedJson(value.answer, ctx) +}) + +const JSON_VALUE_COUNT_LIMIT = 50_000 + +function validateBoundedJson(value: unknown, ctx: z.RefinementCtx): void { + const issue = inspectBoundedJson(value) + if (issue) ctx.addIssue({ code: z.ZodIssueCode.custom, message: issue }) +} + +function inspectBoundedJson(input: unknown): string | undefined { + const pending: Array<{ value: unknown; depth: number }> = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let count = 0 + let bytes = 0 + + while (pending.length) { + const { value, depth } = pending.pop()! + if (++count > JSON_VALUE_COUNT_LIMIT) return "JSON value contains too many values" + if (depth > WORKFLOW_LIMITS.valueDepth) return "JSON value is too deeply nested" + if (value === null) bytes += 4 + else if (typeof value === "string") bytes += Buffer.byteLength(JSON.stringify(value), "utf8") + else if (typeof value === "boolean") bytes += value ? 4 : 5 + else if (typeof value === "number" && Number.isFinite(value)) bytes += JSON.stringify(value).length + else if (value && typeof value === "object") { + if (seen.has(value)) return "JSON value must not contain cycles or aliases" + seen.add(value) + if (Array.isArray(value)) { + const keys = Object.keys(value) + if (keys.length !== value.length || keys.some((key, index) => key !== String(index))) { + return "JSON value must contain only plain arrays" + } + bytes += 2 + Math.max(0, value.length - 1) + for (let index = value.length - 1; index >= 0; index--) { + pending.push({ value: value[index], depth: depth + 1 }) + } + } else { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return "JSON value must contain only plain objects" + const keys = Object.keys(value) + if (Reflect.ownKeys(value).length !== keys.length) return "JSON value must contain only plain objects" + bytes += 2 + Math.max(0, keys.length - 1) + for (const key of keys) { + const descriptor = Object.getOwnPropertyDescriptor(value, key) + if (!descriptor || !("value" in descriptor) || !descriptor.enumerable) return "JSON value must contain only plain objects" + bytes += Buffer.byteLength(JSON.stringify(key), "utf8") + 1 + pending.push({ value: descriptor.value, depth: depth + 1 }) + } + } + } else return "JSON value must contain only JSON values" + + if (bytes > WORKFLOW_LIMITS.sourceBytes) return "JSON value is too large" + } +} + +const definitionSource = (value: { source?: string; definition?: unknown }) => value.source ?? value.definition +const belongsToWorkspace = (run: { workspaceId: string; worktreeSelection?: { sourceWorkspaceId: string } }, workspaceId: string) => + run.workspaceId === workspaceId || run.worktreeSelection?.sourceWorkspaceId === workspaceId + +const handleWorkflowError = (error: unknown, reply: { code(statusCode: number): unknown }) => { + if (error instanceof WorkflowRunError || error instanceof WorkflowDefinitionStoreError) { + reply.code(error.statusCode) + return { error: error.message } + } + if (error instanceof z.ZodError) { + reply.code(400) + return { error: "Invalid workflow definition", issues: error.flatten() } + } + throw error +} + +async function startLatestDefinition( + manager: WorkflowManagerWithLatestStart, + input: WorkflowDefinitionRunCreateRequest, +): Promise { + if (!manager.startLatest) { + throw new WorkflowRunError("Atomic saved workflow start is unavailable", 501) + } + return manager.startLatest(input) +} + +export function registerWorkflowRoutes(app: FastifyInstance, deps: RouteDeps) { + app.get<{ Params: { id: string } }>("/workspaces/:id/plugin/workflow-definitions", async () => { + return { definitions: await deps.workflowManager.listDefinitions() } + }) + + app.get<{ Params: { id: string; definitionId: string } }>( + "/workspaces/:id/plugin/workflow-definitions/:definitionId", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + if (!id.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + const definition = await deps.workflowManager.getDefinition(id.data) + if (!definition) { + reply.code(404) + return { error: "Workflow definition not found" } + } + return definition + }, + ) + + app.post<{ Params: { id: string; definitionId: string } }>( + "/workspaces/:id/plugin/workflow-definitions/:definitionId/start", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const body = PluginDefinitionStartSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow request" } + } + try { + const run = await deps.workflowManager.start({ + ...body.data, + workspaceId: request.params.id, + definitionId: id.data, + }) + reply.code(202) + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }, + ) + + app.get<{ Params: { id: string } }>("/workspaces/:id/plugin/workflow-runs", async (request) => { + return { runs: await deps.workflowManager.list(request.params.id) } + }) + + app.get<{ Params: { id: string; runId: string } }>( + "/workspaces/:id/plugin/workflow-runs/:runId", + async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + const run = await deps.workflowManager.get(parsed.data, request.params.id) + if (!run || !belongsToWorkspace(run, request.params.id)) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + }, + ) + + app.post<{ Params: { id: string; runId: string } }>( + "/workspaces/:id/plugin/workflow-runs/:runId/cancel", + async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + try { + const run = await deps.workflowManager.cancelOwned(parsed.data, request.params.id) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }, + ) + + app.post("/api/workflow-definitions/validate", async (request, reply) => { + const parsed = DefinitionSourceSchema.safeParse(request.body) + if (!parsed.success) { + reply.code(400) + return { valid: false, issues: parsed.error.issues } + } + const result = deps.workflowManager.validateDefinition(definitionSource(parsed.data)) + if (!result.valid) reply.code(400) + return result + }) + + app.get("/api/workflow-definitions", async () => ({ definitions: await deps.workflowManager.listDefinitions() })) + + app.post("/api/workflow-definitions", async (request, reply) => { + const parsed = DefinitionSourceSchema.safeParse(request.body) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow definition request", issues: parsed.error.flatten() } + } + try { + const record = await deps.workflowManager.createDefinition(definitionSource(parsed.data)) + reply.code(201) + return record + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.get<{ Params: { definitionId: string }; Querystring: { revision?: string } }>( + "/api/workflow-definitions/:definitionId", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const query = DefinitionRevisionQuerySchema.safeParse(request.query) + if (!id.success || !query.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + const definition = await deps.workflowManager.getDefinition(id.data, query.data.revision) + if (!definition) { + reply.code(404) + return { error: "Workflow definition not found" } + } + return definition + }, + ) + + app.put<{ Params: { definitionId: string } }>("/api/workflow-definitions/:definitionId", async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const body = DefinitionUpdateSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + try { + return await deps.workflowManager.updateDefinition(id.data, body.data.expectedRevision, definitionSource(body.data)) + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.delete<{ Params: { definitionId: string }; Querystring: { expectedRevision?: string } }>( + "/api/workflow-definitions/:definitionId", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const query = DeleteDefinitionQuerySchema.safeParse(request.query) + if (!id.success || !query.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + try { + const deleted = await deps.workflowManager.deleteDefinition(id.data, query.data.expectedRevision) + if (!deleted) { + reply.code(404) + return { error: "Workflow definition not found" } + } + reply.code(204) + return undefined + } catch (error) { + return handleWorkflowError(error, reply) + } + }, + ) + + app.post<{ Params: { definitionId: string } }>("/api/workflow-definitions/:definitionId/start", async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const body = DefinitionStartObjectSchema.omit({ definitionId: true }).safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow request" } + } + try { + const run = await startLatestDefinition(deps.workflowManager, { ...body.data, definitionId: id.data }) + reply.code(202) + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.get("/api/workflow-runs", async (request, reply) => { + const parsed = ListSchema.safeParse(request.query) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow query" } + } + return { runs: await deps.workflowManager.list(parsed.data.workspaceId) } + }) + + app.post("/api/workflow-runs", async (request, reply) => { + const parsed = CreateSchema.safeParse(request.body) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow request", issues: parsed.error.flatten() } + } + + try { + const run = "stages" in parsed.data + ? await deps.workflowManager.start(parsed.data) + : await startLatestDefinition(deps.workflowManager, parsed.data) + reply.code(202) + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.get<{ Params: { runId: string } }>("/api/workflow-runs/:runId", async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + const run = await deps.workflowManager.get(parsed.data) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/cancel", async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + try { + const run = await deps.workflowManager.cancel(parsed.data) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/approve", async (request, reply) => { + const id = RunIdSchema.safeParse(request.params.runId) + const body = ApprovalSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow approval request" } + } + try { + const run = await deps.workflowManager.approve(id.data, body.data.expectedStepId) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/pause", async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + try { + const run = await deps.workflowManager.pause(parsed.data) + if (!run) { reply.code(404); return { error: "Workflow run not found" } } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/resume", async (request, reply) => { + const id = RunIdSchema.safeParse(request.params.runId) + const body = ResumeSchema.safeParse(request.body ?? {}) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow resume request" } + } + try { + const run = await deps.workflowManager.resume(id.data, body.data.confirmRecovery) + if (!run) { reply.code(404); return { error: "Workflow run not found" } } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/answer", async (request, reply) => { + const id = RunIdSchema.safeParse(request.params.runId) + const body = AnswerSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow gate answer" } + } + try { + const run = await deps.workflowManager.answer(id.data, body.data.executionNodeId, body.data.answer) + if (!run) { reply.code(404); return { error: "Workflow run not found" } } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) +} diff --git a/packages/server/src/server/routes/workspaces.test.ts b/packages/server/src/server/routes/workspaces.test.ts index e115c5b7a..d12a0a423 100644 --- a/packages/server/src/server/routes/workspaces.test.ts +++ b/packages/server/src/server/routes/workspaces.test.ts @@ -3,7 +3,7 @@ import { describe, it } from "node:test" import Fastify from "fastify" import type { WorkspaceDescriptor } from "../../api-types" -import type { WorkspaceManager } from "../../workspaces/manager" +import { WorkspaceDeletionBlockedError, type WorkspaceManager } from "../../workspaces/manager" import { registerWorkspaceRoutes } from "./workspaces" describe("workspace routes", () => { @@ -40,6 +40,7 @@ describe("workspace routes", () => { path: "C:/work", name: "Work", binaryPath: " C:/tools/opencode.exe ", + lineageId: "00000000-0000-4000-8000-000000000001", requestId: " restore-request ", forceNew: true, }, @@ -48,6 +49,7 @@ describe("workspace routes", () => { assert.equal(response.statusCode, 201) assert.deepEqual(calls, [["C:/work", "Work", { binaryPath: "C:/tools/opencode.exe", + lineageId: "00000000-0000-4000-8000-000000000001", requestId: "restore-request", forceNew: true, }]]) @@ -123,4 +125,30 @@ describe("workspace routes", () => { assert.equal((await cancellation).statusCode, 204) await app.close() }) + + it("reports workflow-owned deletion and restore cancellation as conflicts", async () => { + let deleted = false + const app = Fastify({ logger: false }) + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/worktree" }), + delete: async () => { + deleted = true + throw new WorkspaceDeletionBlockedError("workspace") + }, + cancelCreationRequest: async () => { throw new WorkspaceDeletionBlockedError("workspace") }, + } as unknown as WorkspaceManager + registerWorkspaceRoutes(app, { workspaceManager }) + + const response = await app.inject({ method: "DELETE", url: "/api/workspaces/workspace" }) + const cancellation = await app.inject({ + method: "POST", + url: "/api/workspaces/creation/cancel", + payload: { requestId: "restore-request" }, + }) + + assert.equal(response.statusCode, 409) + assert.equal(cancellation.statusCode, 409) + assert.equal(deleted, true) + await app.close() + }) }) diff --git a/packages/server/src/server/routes/workspaces.ts b/packages/server/src/server/routes/workspaces.ts index e7f052136..ac334acd0 100644 --- a/packages/server/src/server/routes/workspaces.ts +++ b/packages/server/src/server/routes/workspaces.ts @@ -1,6 +1,6 @@ import { FastifyInstance, FastifyReply } from "fastify" import { z } from "zod" -import { WorkspaceManager } from "../../workspaces/manager" +import { WorkspaceDeletionBlockedError, WorkspaceManager } from "../../workspaces/manager" import { getWorktreeGitDiff, getWorktreeGitStatus } from "../../workspaces/git-status" import { commitWorktreeChanges, isGitMutationError, stageWorktreePaths, unstageWorktreePaths } from "../../workspaces/git-mutations" import { cloneGitRepository, isGitCloneError } from "../../workspaces/git-clone" @@ -13,6 +13,7 @@ interface RouteDeps { const WorkspaceCreateSchema = z.object({ path: z.string(), + lineageId: z.string().uuid().optional(), name: z.string().optional(), binaryPath: z.string().trim().min(1).max(4096).optional(), requestId: z.string().trim().min(1).max(128).optional(), @@ -79,6 +80,7 @@ export function registerWorkspaceRoutes(app: FastifyInstance, deps: RouteDeps) { binaryPath: body.binaryPath, requestId: body.requestId, forceNew: body.forceNew, + ...(body.lineageId ? { lineageId: body.lineageId } : {}), }) reply.code(201) return result.created ? result.workspace : { ...result.workspace, reused: true as const } @@ -110,8 +112,12 @@ export function registerWorkspaceRoutes(app: FastifyInstance, deps: RouteDeps) { }) app.delete<{ Params: { id: string } }>("/api/workspaces/:id", async (request, reply) => { - await deps.workspaceManager.delete(request.params.id) - reply.code(204) + try { + await deps.workspaceManager.delete(request.params.id) + reply.code(204) + } catch (error) { + return handleWorkspaceError(error, reply) + } }) app.post("/api/workspaces/creation/cancel", async (request, reply) => { @@ -120,8 +126,12 @@ export function registerWorkspaceRoutes(app: FastifyInstance, deps: RouteDeps) { reply.code(400).type("text/plain").send("Invalid workspace creation request") return } - await deps.workspaceManager.cancelCreationRequest(parsed.data.requestId) - reply.code(204) + try { + await deps.workspaceManager.cancelCreationRequest(parsed.data.requestId) + reply.code(204) + } catch (error) { + return handleWorkspaceError(error, reply) + } }) app.post<{ Params: { id: string } }>("/api/workspaces/:id/creation/release", async (request, reply) => { @@ -331,6 +341,10 @@ async function resolveGitWorktreeDirectory( function handleWorkspaceError(error: unknown, reply: FastifyReply) { + if (error instanceof WorkspaceDeletionBlockedError) { + reply.code(409) + return { error: error.message } + } if (isGitCloneError(error)) { reply.code(error.statusCode) return { error: error.message } diff --git a/packages/server/src/server/routes/worktrees.test.ts b/packages/server/src/server/routes/worktrees.test.ts new file mode 100644 index 000000000..45adece49 --- /dev/null +++ b/packages/server/src/server/routes/worktrees.test.ts @@ -0,0 +1,96 @@ +import assert from "node:assert/strict" +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import { spawnSync } from "node:child_process" +import { test } from "node:test" +import Fastify from "fastify" + +import { createManagedWorktree } from "../../workspaces/git-worktrees" +import { registerWorktreeRoutes } from "./worktrees" + +test("managed worktree deletion rejects workflow ownership and live workspaces", async (context) => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-worktree-route-")) + const git = (...args: string[]) => spawnSync("git", args, { cwd: temp, encoding: "utf8" }) + if (git("--version").error) { + context.skip("Git is unavailable") + rmSync(temp, { recursive: true, force: true }) + return + } + try { + assert.equal(git("init").status, 0) + assert.equal(git("config", "user.email", "test@example.com").status, 0) + assert.equal(git("config", "user.name", "CodeNomad Test").status, 0) + writeFileSync(path.join(temp, "file.txt"), "initial") + assert.equal(git("add", "file.txt").status, 0) + assert.equal(git("commit", "-m", "initial").status, 0) + const worktree = await createManagedWorktree({ repoRoot: temp, workspaceFolder: temp, slug: "review" }) + + const app = Fastify({ logger: false }) + registerWorktreeRoutes(app, { + workspaceManager: { get: () => ({ id: "workspace", path: temp }), list: () => [] } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + list: async () => { throw new Error("capped list must not be used") }, + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(true), + } as never, + }) + + const response = await app.inject({ method: "DELETE", url: "/api/workspaces/workspace/worktrees/review" }) + + assert.equal(response.statusCode, 409) + assert.equal(existsSync(worktree.directory), true) + await app.close() + + const workspaceApp = Fastify({ logger: false }) + registerWorktreeRoutes(workspaceApp, { + workspaceManager: { + get: (id: string) => id === "workspace" + ? { id, path: temp } + : id === "execution" ? { id, path: worktree.directory, status: "ready" } : undefined, + list: () => [{ id: "execution", path: path.join(worktree.directory, ".") }], + } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(false), + } as never, + }) + + const activeWorkspace = await workspaceApp.inject({ + method: "DELETE", + url: "/api/workspaces/workspace/worktrees/review", + }) + assert.equal(activeWorkspace.statusCode, 409) + assert.match(activeWorkspace.body, /active workspace/) + assert.equal(existsSync(worktree.directory), true) + await workspaceApp.close() + + const terminalWorkspaceApp = Fastify({ logger: false }) + registerWorktreeRoutes(terminalWorkspaceApp, { + workspaceManager: { + get: (id: string) => id === "workspace" + ? { id, path: temp } + : id === "stopped" ? { id, path: worktree.directory, status: "stopped" } + : id === "error" ? { id, path: worktree.directory, status: "error" } : undefined, + list: () => [ + { id: "stopped", path: worktree.directory, status: "stopped" }, + { id: "error", path: worktree.directory, status: "error" }, + ], + } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(false), + } as never, + }) + + const terminalWorkspaces = await terminalWorkspaceApp.inject({ + method: "DELETE", + url: "/api/workspaces/workspace/worktrees/review", + }) + assert.equal(terminalWorkspaces.statusCode, 204) + assert.equal(existsSync(worktree.directory), false) + await terminalWorkspaceApp.close() + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) diff --git a/packages/server/src/server/routes/worktrees.ts b/packages/server/src/server/routes/worktrees.ts index d48d9ddb1..17bc033b1 100644 --- a/packages/server/src/server/routes/worktrees.ts +++ b/packages/server/src/server/routes/worktrees.ts @@ -5,16 +5,20 @@ import { resolveRepoRoot, listWorktrees, isValidWorktreeSlug, + isManagedWorktree, createManagedWorktree, removeWorktree, } from "../../workspaces/git-worktrees" import type { WorktreeListResponse, WorktreeMap } from "../../api-types" import type { OpencodeYoloPersistence } from "../../permissions/opencode-yolo-metadata" import { ensureCodenomadGitExclude, readWorktreeMap, writeWorktreeMap } from "../../workspaces/worktree-map" +import { resolveWorkspaceIdentity } from "../../workspaces/workspace-identity" +import type { WorkflowManager } from "../../workflows/manager" interface RouteDeps { workspaceManager: WorkspaceManager sessionMetadataPersistence: OpencodeYoloPersistence + workflowManager: WorkflowManager } const WorktreeMapSchema = z.object({ @@ -146,36 +150,60 @@ export function registerWorktreeRoutes(app: FastifyInstance, deps: RouteDeps) { reply.code(404) return { error: "Worktree not found" } } - - await removeWorktree({ workspaceFolder: workspace.path, directory: match.directory, force, logger: request.log }) - - // Best-effort: prune any mappings that point at the deleted worktree. - const current = await readWorktreeMap(workspace.path, request.log) - let changed = false - const nextMapping: Record = { ...(current.parentSessionWorktreeSlug ?? {}) } - for (const [sessionId, mapped] of Object.entries(nextMapping)) { - if (mapped === slug) { - delete nextMapping[sessionId] - changed = true - } - } - const nextDefault = current.defaultWorktreeSlug === slug ? "root" : current.defaultWorktreeSlug - if (nextDefault !== current.defaultWorktreeSlug) { - changed = true - } - if (changed) { - await writeWorktreeMap( - workspace.path, - { - version: 1, - defaultWorktreeSlug: nextDefault, - parentSessionWorktreeSlug: nextMapping, - }, - request.log, - ) + if (!await isManagedWorktree({ repoRoot, worktree: match })) { + reply.code(404) + return { error: "Managed worktree not found" } } - - reply.code(204) + return await deps.workflowManager.withWorktreeOwnershipLease( + { id: workspace.id, lineageId: workspace.lineageId, path: workspace.path }, + { slug, path: match.directory }, + async (owned) => { + if (owned) { + reply.code(409) + return { error: "Worktree is owned by an active workflow" } + } + const targetIdentity = await resolveWorkspaceIdentity(match.directory, process.cwd()) + for (const listed of deps.workspaceManager.list()) { + const live = deps.workspaceManager.get(listed.id) + if (!live) continue + if (live.status !== "starting" && live.status !== "ready") continue + const liveIdentity = await resolveWorkspaceIdentity(live.path, process.cwd()) + if (liveIdentity.identityKey !== targetIdentity.identityKey) continue + reply.code(409) + return { error: "Worktree is in use by an active workspace" } + } + + await removeWorktree({ workspaceFolder: workspace.path, directory: match.directory, force, logger: request.log }) + + // Best-effort: prune any mappings that point at the deleted worktree. + const current = await readWorktreeMap(workspace.path, request.log) + let changed = false + const nextMapping: Record = { ...(current.parentSessionWorktreeSlug ?? {}) } + for (const [sessionId, mapped] of Object.entries(nextMapping)) { + if (mapped === slug) { + delete nextMapping[sessionId] + changed = true + } + } + const nextDefault = current.defaultWorktreeSlug === slug ? "root" : current.defaultWorktreeSlug + if (nextDefault !== current.defaultWorktreeSlug) { + changed = true + } + if (changed) { + await writeWorktreeMap( + workspace.path, + { + version: 1, + defaultWorktreeSlug: nextDefault, + parentSessionWorktreeSlug: nextMapping, + }, + request.log, + ) + } + + reply.code(204) + } + ) } catch (error) { return handleError(error, reply) } diff --git a/packages/server/src/shutdown.test.ts b/packages/server/src/shutdown.test.ts index 446e77918..befc62987 100644 --- a/packages/server/src/shutdown.test.ts +++ b/packages/server/src/shutdown.test.ts @@ -10,7 +10,7 @@ import { const logger = { info() {}, warn() {}, error() {} } const operations = (overrides: Partial = {}): ServerShutdownOperations => ({ - stopInstanceEventBridge() {}, stopSidecars() {}, stopClientConnections() {}, + stopInstanceEventBridge() {}, stopWorkflowRuns() {}, stopSidecars() {}, stopClientConnections() {}, stopRemoteProxySessions() {}, stopWorkspaces() {}, stopHttpServers() {}, stopReleaseMonitor() {}, ...overrides, }) @@ -24,7 +24,9 @@ describe("server shutdown orchestration", () => { stopWorkspaces: () => { calls.push(`workspaces-${++attempts}`); if (attempts === 1) throw new Error("still alive") }, stopHttpServers: () => { calls.push("http") }, }), logger) - assert.deepEqual(calls, ["workspaces-1", "remote-proxy", "workspaces-2", "http"]) + assert.ok(calls.indexOf("remote-proxy") < calls.indexOf("http")) + assert.ok(calls.indexOf("workspaces-1") < calls.indexOf("workspaces-2")) + assert.ok(calls.indexOf("workspaces-2") < calls.indexOf("http")) }) it("closes remaining resources and aggregates the concrete current error", async () => { @@ -45,20 +47,35 @@ describe("server shutdown orchestration", () => { assert.deepEqual([attempts, closed], [2, ["http", "release-monitor"]]) }) - it("starts workspace cleanup without waiting for preliminary shutdown", async () => { - let releasePreliminary!: () => void - const preliminary = new Promise((resolve) => { releasePreliminary = resolve }) + it("finishes workflow cancellation before workspace cleanup without blocking unrelated cleanup", async () => { + let releaseWorkflow!: () => void + const workflow = new Promise((resolve) => { releaseWorkflow = resolve }) + let releaseUnrelated!: () => void + const unrelated = new Promise((resolve) => { releaseUnrelated = resolve }) let workspaceStarted = false const shutdown = orchestrateServerShutdown(operations({ - stopRemoteProxySessions: () => preliminary, + stopWorkflowRuns: () => workflow, + stopRemoteProxySessions: () => unrelated, stopWorkspaces: () => { workspaceStarted = true }, }), logger) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(workspaceStarted, false) + releaseWorkflow() await new Promise((resolve) => setImmediate(resolve)) assert.equal(workspaceStarted, true) - releasePreliminary() + releaseUnrelated() await shutdown }) + + it("still cleans workspaces after workflow cancellation fails", async () => { + let workspaceStarted = false + await assert.rejects(orchestrateServerShutdown(operations({ + stopWorkflowRuns: () => { throw new Error("abort failed") }, + stopWorkspaces: () => { workspaceStarted = true }, + }), logger), AggregateError) + assert.equal(workspaceStarted, true) + }) }) describe("server shutdown signal boundary", () => { diff --git a/packages/server/src/shutdown.ts b/packages/server/src/shutdown.ts index 3324aa255..ab4a6a7b1 100644 --- a/packages/server/src/shutdown.ts +++ b/packages/server/src/shutdown.ts @@ -6,7 +6,7 @@ export const SERVER_SHUTDOWN_COMPLETE = "CODENOMAD_SHUTDOWN_STATUS:complete" export const SERVER_SHUTDOWN_INCOMPLETE = "CODENOMAD_SHUTDOWN_STATUS:incomplete" export type ServerShutdownOperations = Record< - "stopInstanceEventBridge" | "stopSidecars" | "stopClientConnections" | "stopRemoteProxySessions" | "stopWorkspaces" | + "stopInstanceEventBridge" | "stopWorkflowRuns" | "stopSidecars" | "stopClientConnections" | "stopRemoteProxySessions" | "stopWorkspaces" | "stopHttpServers" | "stopReleaseMonitor", ShutdownOperation > @@ -75,7 +75,7 @@ export async function orchestrateServerShutdown( } } - const workspaceShutdown = (async () => { + const workspaceShutdown = async () => { const attempts = Math.max(1, Math.floor(workspaceAttempts)) for (let attempt = 1; attempt <= attempts; attempt += 1) { const [result] = await Promise.allSettled([Promise.resolve().then(operations.stopWorkspaces)]) @@ -88,13 +88,13 @@ export async function orchestrateServerShutdown( errors.push(error) logger.error({ err: error, attempts }, "Workspace manager shutdown failed") } - })() + } await Promise.all([ settle([ ["stopInstanceEventBridge", operations.stopInstanceEventBridge], ["stopSidecars", operations.stopSidecars], ["stopClientConnections", operations.stopClientConnections], ["stopRemoteProxySessions", operations.stopRemoteProxySessions], ]), - workspaceShutdown, + settle([["stopWorkflowRuns", operations.stopWorkflowRuns]]).then(workspaceShutdown), ]) await settle([["stopHttpServers", operations.stopHttpServers], ["stopReleaseMonitor", operations.stopReleaseMonitor]]) if (errors.length) throw new AggregateError(errors, "Server shutdown failed") diff --git a/packages/server/src/workflows/definition-schema.test.ts b/packages/server/src/workflows/definition-schema.test.ts new file mode 100644 index 000000000..5a3dc7a30 --- /dev/null +++ b/packages/server/src/workflows/definition-schema.test.ts @@ -0,0 +1,136 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + parseWorkflowDefinition, + validateWorkflowDefinition, + WORKFLOW_DEFINITION_REVISION_LIMIT, + WORKFLOW_LIMITS, +} from "./definition-schema" +import { validateJsonSchemaValue } from "./json-schema" + +describe("workflow definition schema", () => { + it("parses canonical YAML without executable tags", () => { + const parsed = parseWorkflowDefinition(` +version: 1 +id: safe +name: Safe workflow +root: + type: agent + id: work + instructions: Do the work + tools: [read] + outputSchema: + type: object + required: [result] + properties: + result: { type: string } +`) + assert.equal(parsed.definition.root.type, "agent") + assert.equal(parsed.canonical, parseWorkflowDefinition(parsed.canonical).canonical) + assert.equal(validateWorkflowDefinition("value: !!js/function function() {}" ).valid, false) + }) + + it("rejects duplicate IDs and statically excessive dynamic expansion", () => { + const duplicate = validateWorkflowDefinition({ + version: 1, id: "duplicate", name: "Duplicate", + root: { type: "sequence", id: "root", steps: [ + { type: "agent", id: "same", instructions: "One" }, + { type: "agent", id: "same", instructions: "Two" }, + ] }, + }) + assert.equal(duplicate.valid, false) + + const expansion = validateWorkflowDefinition({ + version: 1, id: "large", name: "Large", maxExpandedNodes: WORKFLOW_LIMITS.expandedNodes, + root: { + type: "foreach", id: "outer", item: "outerItem", maxItems: 101, items: [], + body: { + type: "repeat", id: "inner", maxIterations: 100, + body: { type: "agent", id: "work", instructions: "Work" }, + }, + }, + }) + assert.equal(expansion.valid, false) + + assert.equal(validateWorkflowDefinition({ + version: 1, id: "unsafe-retry", name: "Unsafe retry", + root: { type: "shell", id: "deploy", agent: "build", command: "deploy", retry: { maxAttempts: 2 } }, + }).valid, false) + }) + + it("accepts saved workflow calls with bounded input values", () => { + const result = validateWorkflowDefinition({ + version: 1, id: "caller", name: "Caller", + root: { + type: "workflow", id: "nested", definitionId: "saved", definitionRevision: 2, + inputs: { environment: "test", payload: { $ref: "inputs.payload" } }, + }, + }) + assert.equal(result.valid, true) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "bad-caller", name: "Bad caller", + root: { type: "workflow", id: "nested", definitionId: "../saved" }, + }).valid, false) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "future-caller", name: "Future caller", + root: { + type: "workflow", id: "nested", definitionId: "saved", + definitionRevision: WORKFLOW_DEFINITION_REVISION_LIMIT + 1, + }, + }).valid, false) + }) + + it("requires portable lowercase saved definition IDs", () => { + assert.equal(validateWorkflowDefinition({ + version: 1, id: "CaseCollision", name: "Uppercase definition", + root: { type: "agent", id: "work", instructions: "Work" }, + }).valid, false) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "lowercase-id", name: "Lowercase definition", + root: { type: "workflow", id: "CallNode", definitionId: "UppercaseTarget" }, + }).valid, false) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "lowercase-id", name: "Lowercase definition", + root: { type: "workflow", id: "CallNode", definitionId: "lowercase-target" }, + }).valid, true) + }) + + it("accepts only the JSON schema subset enforced at runtime", () => { + const definition = (outputSchema: Record) => ({ + version: 1, id: "schema", name: "Schema", + root: { type: "agent", id: "work", instructions: "Work", outputSchema }, + }) + assert.equal(validateWorkflowDefinition(definition({ + type: "object", + required: ["result"], + properties: { result: { type: "string", minLength: 1 } }, + additionalProperties: false, + })).valid, true) + assert.equal(validateWorkflowDefinition(definition({ enum: [null, false, 0, "0", [], {}] })).valid, true) + + for (const outputSchema of [ + { type: "string", pattern: "^(a+)+$" }, + { type: "string", format: "email" }, + { type: "object", properties: { value: { $ref: "#/definitions/value" } } }, + { type: "mystery" }, + { items: true }, + { additionalProperties: { type: "string" } }, + { enum: [] }, + { enum: ["same", "same"] }, + { enum: [{ left: 1, right: 2 }, { right: 2, left: 1 }] }, + ]) assert.equal(validateWorkflowDefinition(definition(outputSchema)).valid, false) + }) + + it("enforces closed objects even when no properties are declared", () => { + const closed = { type: "object", additionalProperties: false } + assert.deepEqual(validateJsonSchemaValue({}, closed), []) + assert.deepEqual(validateJsonSchemaValue({ unexpected: true }, closed), ["$.unexpected is not allowed"]) + assert.deepEqual(validateJsonSchemaValue({ value: 1 }, { + type: "object", properties: { value: { type: "integer" } }, additionalProperties: false, + }), []) + assert.deepEqual(validateJsonSchemaValue({ ordered: { right: 2, left: 1 } }, { + const: { ordered: { left: 1, right: 2 } }, + }), []) + assert.deepEqual(validateJsonSchemaValue("😀", { minLength: 1, maxLength: 1 }), []) + }) +}) diff --git a/packages/server/src/workflows/definition-schema.ts b/packages/server/src/workflows/definition-schema.ts new file mode 100644 index 000000000..c2abee833 --- /dev/null +++ b/packages/server/src/workflows/definition-schema.ts @@ -0,0 +1,277 @@ +import { parseDocument } from "yaml" +import { z } from "zod" +import type { + WorkflowCondition, + WorkflowDefinitionV1, + WorkflowNode, + WorkflowValue, +} from "../api-types" +import { inspectJsonSchema } from "./json-schema" + +export const WORKFLOW_DEFINITION_REVISION_LIMIT = 100 + +export const WORKFLOW_LIMITS = { + sourceBytes: 256_000, + staticNodes: 256, + expandedNodes: 10_000, + depth: 24, + branchWidth: 32, + concurrency: 16, + foreachItems: 1_000, + repeatIterations: 1_000, + retries: 5, + nestingDepth: 8, + timeoutMs: 24 * 60 * 60 * 1_000, + schemaBytes: 32_000, + valueDepth: 20, +} as const + +const IdSchema = z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/) +const DefinitionIdSchema = z.string().trim().min(1).max(100).regex( + /^[a-z0-9][a-z0-9_-]*$/, + "Definition IDs must start with a lowercase letter or number and contain only lowercase letters, numbers, hyphens, or underscores", +) +const TitleSchema = z.string().trim().min(1).max(200) +const ModelSchema = z.object({ + providerID: z.string().trim().min(1).max(200), + modelID: z.string().trim().min(1).max(200), +}).strict() +const RefSchema = z.object({ + $ref: z.string().regex(/^(inputs|nodes|vars)(?:\.[a-zA-Z0-9_-]+)+$/).max(500), +}).strict() + +export const WorkflowValueSchema: z.ZodType = z.lazy(() => z.union([ + z.null(), + z.boolean(), + z.number().finite(), + z.string().max(50_000), + RefSchema, + z.array(WorkflowValueSchema).max(WORKFLOW_LIMITS.foreachItems), + z.record(WorkflowValueSchema), +])) + +export const WorkflowConditionSchema: z.ZodType = z.union([ + z.boolean(), + z.object({ + value: WorkflowValueSchema, + equals: WorkflowValueSchema.optional(), + notEquals: WorkflowValueSchema.optional(), + exists: z.boolean().optional(), + truthy: z.boolean().optional(), + }).strict().superRefine((condition, ctx) => { + const operators = [condition.equals, condition.notEquals, condition.exists, condition.truthy] + .filter((value) => value !== undefined) + if (operators.length > 1) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Condition accepts one operator" }) + }), +]) + +const RetrySchema = z.object({ + maxAttempts: z.number().int().min(1).max(WORKFLOW_LIMITS.retries), + delayMs: z.number().int().min(0).max(60_000).optional(), + idempotent: z.boolean().optional(), +}).strict().superRefine((retry, ctx) => { + if (retry.maxAttempts > 1 && retry.idempotent !== true) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Retries require idempotent: true" }) + } +}) + +const JsonSchema = z.record(z.unknown()).superRefine((value, ctx) => { + try { + const serialized = JSON.stringify(value) + if (Buffer.byteLength(serialized, "utf8") > WORKFLOW_LIMITS.schemaBytes) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "JSON schema is too large" }) + } + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "JSON schema must be JSON serializable" }) + } + for (const issue of inspectJsonSchema(value)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: issue.message, path: issue.path }) + } +}) + +const NodeBase = { + id: IdSchema, + title: TitleSchema.optional(), + if: WorkflowConditionSchema.optional(), +} + +export const WorkflowNodeSchema: z.ZodType = z.lazy(() => z.union([ + z.object({ ...NodeBase, type: z.literal("sequence"), steps: z.array(WorkflowNodeSchema).min(1).max(WORKFLOW_LIMITS.branchWidth) }).strict(), + z.object({ + ...NodeBase, + type: z.literal("parallel"), + branches: z.array(WorkflowNodeSchema).min(1).max(WORKFLOW_LIMITS.branchWidth), + maxConcurrency: z.number().int().min(1).max(WORKFLOW_LIMITS.concurrency).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("foreach"), + items: WorkflowValueSchema, + item: IdSchema, + body: WorkflowNodeSchema, + maxItems: z.number().int().min(1).max(WORKFLOW_LIMITS.foreachItems), + maxConcurrency: z.number().int().min(1).max(WORKFLOW_LIMITS.concurrency).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("repeat"), + body: WorkflowNodeSchema, + maxIterations: z.number().int().min(1).max(WORKFLOW_LIMITS.repeatIterations), + while: WorkflowConditionSchema.optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("agent"), + instructions: z.string().trim().min(1).max(50_000), + context: WorkflowValueSchema.optional(), + agent: z.string().trim().min(1).max(200).optional(), + model: ModelSchema.optional(), + tools: z.array(z.string().trim().min(1).max(200)).max(128).refine((tools) => new Set(tools).size === tools.length, "Tool IDs must be unique").optional(), + outputSchema: JsonSchema.optional(), + retry: RetrySchema.optional(), + timeoutMs: z.number().int().min(1).max(WORKFLOW_LIMITS.timeoutMs).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("shell"), + command: z.string().trim().min(1).max(50_000), + agent: z.string().trim().min(1).max(200), + model: ModelSchema.optional(), + retry: RetrySchema.optional(), + timeoutMs: z.number().int().min(1).max(WORKFLOW_LIMITS.timeoutMs).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("gate"), + gate: z.enum(["approval", "input"]), + prompt: z.string().trim().min(1).max(20_000), + inputSchema: JsonSchema.optional(), + }).strict().superRefine((gate, ctx) => { + if (gate.gate === "approval" && gate.inputSchema) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Approval gates do not accept inputSchema", path: ["inputSchema"] }) + } + }), + z.object({ + ...NodeBase, + type: z.literal("workflow"), + definitionId: DefinitionIdSchema, + definitionRevision: z.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT).optional(), + inputs: z.record(WorkflowValueSchema).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("condition"), + condition: WorkflowConditionSchema, + then: WorkflowNodeSchema, + else: WorkflowNodeSchema.optional(), + }).strict(), +]) as unknown as z.ZodType) + +const DefinitionSchemaBase = z.object({ + version: z.literal(1), + id: DefinitionIdSchema, + name: TitleSchema, + description: z.string().trim().max(2_000).optional(), + root: WorkflowNodeSchema, + budget: z.object({ + maxCost: z.number().finite().positive().max(1_000_000).optional(), + maxTokens: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), + }).strict().refine((budget) => budget.maxCost !== undefined || budget.maxTokens !== undefined, "Budget cannot be empty").optional(), + maxConcurrency: z.number().int().min(1).max(WORKFLOW_LIMITS.concurrency).optional(), + maxExpandedNodes: z.number().int().min(1).max(WORKFLOW_LIMITS.expandedNodes).optional(), +}).strict() + +const inspectTree = (node: WorkflowNode, state: { ids: Set; count: number; estimated: number }, depth: number): string[] => { + const issues: string[] = [] + state.count += 1 + if (depth > WORKFLOW_LIMITS.depth) issues.push(`Node ${node.id} exceeds maximum depth ${WORKFLOW_LIMITS.depth}`) + if (state.ids.has(node.id)) issues.push(`Node ID ${node.id} is duplicated`) + state.ids.add(node.id) + + let children: WorkflowNode[] = [] + let multiplier = 1 + if (node.type === "sequence") children = node.steps + else if (node.type === "parallel") children = node.branches + else if (node.type === "foreach") { children = [node.body]; multiplier = node.maxItems } + else if (node.type === "repeat") { children = [node.body]; multiplier = node.maxIterations } + else if (node.type === "condition") children = [node.then, ...(node.else ? [node.else] : [])] + + const before = state.estimated + state.estimated += 1 + for (const child of children) issues.push(...inspectTree(child, state, depth + 1)) + if (multiplier > 1) state.estimated += (state.estimated - before - 1) * (multiplier - 1) + return issues +} + +export const WorkflowDefinitionSchema: z.ZodType = DefinitionSchemaBase.superRefine((definition, ctx) => { + const state = { ids: new Set(), count: 0, estimated: 0 } + for (const message of inspectTree(definition.root, state, 1)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message, path: ["root"] }) + } + if (state.count > WORKFLOW_LIMITS.staticNodes) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: `Definition exceeds ${WORKFLOW_LIMITS.staticNodes} static nodes`, path: ["root"] }) + } + const expandedLimit = definition.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + if (state.estimated > expandedLimit) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: `Definition can expand to ${state.estimated} nodes, above limit ${expandedLimit}`, path: ["root"] }) + } +}) + +const sortJson = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(sortJson) + if (!value || typeof value !== "object") return value + return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [ + key, + sortJson((value as Record)[key]), + ])) +} + +export interface ParsedWorkflowDefinition { + definition: WorkflowDefinitionV1 + canonical: string +} + +const inspectInput = (input: unknown) => { + const pending = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let values = 0 + while (pending.length) { + const { value, depth } = pending.pop()! + if (++values > 50_000) throw new Error("Workflow definition has too many values") + if (depth > WORKFLOW_LIMITS.depth * 3) throw new Error("Workflow definition is too deeply nested") + if (value === null || typeof value === "string" || typeof value === "boolean") continue + if (typeof value === "number" && Number.isFinite(value)) continue + if (!value || typeof value !== "object") throw new Error("Workflow definition must contain only JSON values") + if (seen.has(value)) throw new Error("Workflow definition cannot contain cycles") + seen.add(value) + if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) { + throw new Error("Workflow definition must contain only plain JSON objects") + } + for (const child of Object.values(value)) pending.push({ value: child, depth: depth + 1 }) + } +} + +export function parseWorkflowDefinition(source: string | unknown): ParsedWorkflowDefinition { + let input = source + if (typeof source === "string") { + if (Buffer.byteLength(source, "utf8") > WORKFLOW_LIMITS.sourceBytes) throw new Error("Workflow definition source is too large") + const document = parseDocument(source, { schema: "core", uniqueKeys: true }) + if (document.errors.length) throw new Error(document.errors.map((error) => error.message).join("; ")) + input = document.toJS({ maxAliasCount: 0 }) + } + inspectInput(input) + if (Buffer.byteLength(JSON.stringify(input), "utf8") > WORKFLOW_LIMITS.sourceBytes) { + throw new Error("Workflow definition source is too large") + } + const definition = WorkflowDefinitionSchema.parse(input) + return { definition, canonical: `${JSON.stringify(sortJson(definition), null, 2)}\n` } +} + +export function validateWorkflowDefinition(source: string | unknown) { + try { + return { valid: true as const, ...parseWorkflowDefinition(source) } + } catch (error) { + if (error instanceof z.ZodError) return { valid: false as const, issues: error.issues } + return { valid: false as const, issues: [{ code: "custom", path: [], message: error instanceof Error ? error.message : String(error) }] } + } +} diff --git a/packages/server/src/workflows/definition-store.test.ts b/packages/server/src/workflows/definition-store.test.ts new file mode 100644 index 000000000..0aa02030e --- /dev/null +++ b/packages/server/src/workflows/definition-store.test.ts @@ -0,0 +1,125 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { + WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT, + WORKFLOW_DEFINITION_RECORD_LIMIT, + WORKFLOW_DEFINITION_REVISION_LIMIT, + WorkflowDefinitionStore, +} from "./definition-store" + +const definition = (name: string) => ({ + version: 1 as const, + id: "stored", + name, + root: { type: "agent" as const, id: "work", instructions: "Work" }, +}) + +describe("WorkflowDefinitionStore", () => { + it("rejects definition IDs that can collide by case", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-case-")) + try { + const store = new WorkflowDefinitionStore(directory) + await assert.rejects(store.create({ ...definition("Uppercase"), id: "Stored" }), /lowercase/) + assert.equal((await store.create(definition("Lowercase"))).id, "stored") + await assert.rejects(store.get("Stored"), /Invalid workflow definition ID/) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps immutable revisions and atomically persists a tombstone", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-")) + try { + const store = new WorkflowDefinitionStore(directory) + const first = await store.create(definition("First")) + const second = await store.update("stored", 1, definition("Second")) + assert.equal(first.revision, 1) + assert.equal(second.revision, 2) + assert.equal((await store.get("stored", 1))?.definition.name, "First") + assert.equal((await store.get("stored"))?.definition.name, "Second") + await assert.rejects(store.update("stored", 1, definition("Stale")), /revision is 2/) + assert.equal(await store.delete("stored", 2), true) + assert.equal(await store.get("stored"), undefined) + assert.equal(await store.get("stored", 1), undefined) + assert.equal((await store.inspectRevision("stored", 1))?.definition.name, "First") + assert.equal((await store.inspectRevision("stored", 2))?.definition.name, "Second") + assert.deepEqual((await fs.readdir(directory)).filter((entry) => entry.endsWith(".tmp")), []) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("bounds revision count without pruning immutable or tombstoned history", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-count-")) + try { + const store = new WorkflowDefinitionStore(directory) + await store.create(definition("Revision 1")) + for (let revision = 2; revision <= WORKFLOW_DEFINITION_REVISION_LIMIT; revision++) { + await store.update("stored", revision - 1, definition(`Revision ${revision}`)) + } + await assert.rejects( + store.update("stored", WORKFLOW_DEFINITION_REVISION_LIMIT, definition("Over limit")), + /revision limit reached/, + ) + assert.equal((await store.get("stored"))?.revision, WORKFLOW_DEFINITION_REVISION_LIMIT) + assert.equal((await store.inspectRevision("stored", 1))?.definition.name, "Revision 1") + assert.equal(await store.delete("stored", WORKFLOW_DEFINITION_REVISION_LIMIT), true) + assert.equal(await store.get("stored"), undefined) + assert.equal((await store.inspectRevision("stored", WORKFLOW_DEFINITION_REVISION_LIMIT))?.definition.name, `Revision ${WORKFLOW_DEFINITION_REVISION_LIMIT}`) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("bounds aggregate revision bytes without blocking tombstones", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-bytes-")) + try { + const store = new WorkflowDefinitionStore(directory) + const largeDefinition = (name: string) => ({ + version: 1 as const, + id: "stored", + name, + root: { + type: "sequence" as const, + id: "root", + steps: Array.from({ length: 5 }, (_, index) => ({ + type: "agent" as const, + id: `work-${index}`, + instructions: `${index}${"x".repeat(49_000)}`, + })), + }, + }) + let revision = (await store.create(largeDefinition("Large 1"))).revision + await assert.rejects(async () => { + while (revision < WORKFLOW_DEFINITION_REVISION_LIMIT) { + revision = (await store.update("stored", revision, largeDefinition(`Large ${revision + 1}`))).revision + } + }, /history size limit reached/) + assert.ok(revision < WORKFLOW_DEFINITION_REVISION_LIMIT) + const storedPath = path.join(directory, "stored.json") + const persisted = JSON.parse(await fs.readFile(storedPath, "utf8")) as { revisions: unknown[] } + assert.ok(persisted.revisions.reduce((total, record) => total + Buffer.byteLength(JSON.stringify(record), "utf8"), 0) <= WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT) + assert.equal((await store.get("stored"))?.revision, revision) + assert.equal((await store.inspectRevision("stored", 1))?.definition.name, "Large 1") + assert.equal(await store.delete("stored", revision), true) + assert.equal((await store.inspectRevision("stored", revision))?.revision, revision) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("bounds active and tombstoned definition records", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-records-")) + try { + await Promise.all(Array.from({ length: WORKFLOW_DEFINITION_RECORD_LIMIT }, (_, index) => + fs.writeFile(path.join(directory, `old-${index}.json`), "{}"))) + const store = new WorkflowDefinitionStore(directory) + await assert.rejects(store.create(definition("Over limit")), /record limit reached/) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workflows/definition-store.ts b/packages/server/src/workflows/definition-store.ts new file mode 100644 index 000000000..703010d52 --- /dev/null +++ b/packages/server/src/workflows/definition-store.ts @@ -0,0 +1,217 @@ +import { randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import path from "node:path" +import type { WorkflowDefinitionRecord } from "../api-types" +import { parseWorkflowDefinition, WORKFLOW_DEFINITION_REVISION_LIMIT } from "./definition-schema" + +export { WORKFLOW_DEFINITION_REVISION_LIMIT } from "./definition-schema" + +interface StoredDefinition { + version: 1 + id: string + currentRevision: number + deletedAt?: string + revisions: WorkflowDefinitionRecord[] +} + +export class WorkflowDefinitionStoreError extends Error { + constructor(message: string, readonly statusCode: number) { + super(message) + } +} + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T +export const WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT = 4 * 1024 * 1024 +export const WORKFLOW_DEFINITION_RECORD_LIMIT = 1_000 + +const revisionBytes = (record: WorkflowDefinitionRecord) => Buffer.byteLength(JSON.stringify(record), "utf8") + +function assertHistoryCapacity(revisions: WorkflowDefinitionRecord[], next: WorkflowDefinitionRecord): void { + if (revisions.length >= WORKFLOW_DEFINITION_REVISION_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition revision limit reached", 409) + } + const bytes = revisions.reduce((total, record) => total + revisionBytes(record), revisionBytes(next)) + if (bytes > WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition history size limit reached", 409) + } +} + +export class WorkflowDefinitionStore { + private queue = Promise.resolve() + + constructor(private readonly directory: string) {} + + async create(source: string | unknown): Promise { + return this.write(async () => { + const parsed = this.parse(source) + if (await this.readFile(parsed.definition.id)) { + throw new WorkflowDefinitionStoreError("Workflow definition already exists", 409) + } + await this.assertCreateCapacity() + const now = new Date().toISOString() + const record: WorkflowDefinitionRecord = { + id: parsed.definition.id, + revision: 1, + definition: parsed.definition, + canonical: parsed.canonical, + createdAt: now, + updatedAt: now, + } + assertHistoryCapacity([], record) + await this.persist({ version: 1, id: record.id, currentRevision: 1, revisions: [record] }) + return clone(record) + }) + } + + async update(id: string, expectedRevision: number, source: string | unknown): Promise { + return this.write(async () => { + const stored = await this.requireCurrent(id) + if (stored.currentRevision !== expectedRevision) { + throw new WorkflowDefinitionStoreError(`Workflow definition revision is ${stored.currentRevision}`, 409) + } + const parsed = this.parse(source) + if (parsed.definition.id !== id) throw new WorkflowDefinitionStoreError("Definition ID cannot be changed", 400) + const previous = stored.revisions.at(-1)! + const record: WorkflowDefinitionRecord = { + id, + revision: expectedRevision + 1, + definition: parsed.definition, + canonical: parsed.canonical, + createdAt: previous.createdAt, + updatedAt: new Date().toISOString(), + } + assertHistoryCapacity(stored.revisions, record) + stored.currentRevision = record.revision + stored.revisions.push(record) + await this.persist(stored) + return clone(record) + }) + } + + async delete(id: string, expectedRevision: number): Promise { + return this.write(async () => { + const stored = await this.readFile(id) + if (!stored || stored.deletedAt) return false + if (stored.currentRevision !== expectedRevision) { + throw new WorkflowDefinitionStoreError(`Workflow definition revision is ${stored.currentRevision}`, 409) + } + stored.deletedAt = new Date().toISOString() + await this.persist(stored) + return true + }) + } + + async get(id: string, revision?: number): Promise { + await this.queue.catch(() => undefined) + const stored = await this.readFile(id) + if (!stored || stored.deletedAt) return undefined + return this.selectRevision(stored, revision) + } + + async inspectRevision(id: string, revision: number): Promise { + await this.queue.catch(() => undefined) + const stored = await this.readFile(id) + if (!stored) return undefined + return this.selectRevision(stored, revision) + } + + private selectRevision(stored: StoredDefinition, revision?: number): WorkflowDefinitionRecord | undefined { + const selected = revision === undefined + ? stored.revisions.find((record) => record.revision === stored.currentRevision) + : stored.revisions.find((record) => record.revision === revision) + return selected ? clone(selected) : undefined + } + + async list(): Promise { + await this.queue.catch(() => undefined) + let entries: string[] + try { + entries = await fs.readdir(this.directory) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return [] + throw error + } + const records = await Promise.all(entries.filter((entry) => entry.endsWith(".json")).map(async (entry) => { + const stored = await this.readFile(entry.slice(0, -5)) + if (!stored || stored.deletedAt) return undefined + return stored.revisions.find((record) => record.revision === stored.currentRevision) + })) + return records.filter((record): record is WorkflowDefinitionRecord => Boolean(record)) + .sort((left, right) => left.definition.name.localeCompare(right.definition.name)) + .map(clone) + } + + private async requireCurrent(id: string): Promise { + const stored = await this.readFile(id) + if (!stored || stored.deletedAt) throw new WorkflowDefinitionStoreError("Workflow definition not found", 404) + return stored + } + + private async assertCreateCapacity(): Promise { + let entries: string[] + try { + entries = await fs.readdir(this.directory) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + if (entries.filter((entry) => entry.endsWith(".json")).length >= WORKFLOW_DEFINITION_RECORD_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition record limit reached", 409) + } + } + + private async write(operation: () => Promise): Promise { + const pending = this.queue.catch(() => undefined).then(operation) + this.queue = pending.then(() => undefined, () => undefined) + return pending + } + + private definitionPath(id: string) { + if (!/^[a-z0-9][a-z0-9_-]{0,99}$/.test(id)) throw new WorkflowDefinitionStoreError("Invalid workflow definition ID", 400) + return path.join(this.directory, `${id}.json`) + } + + private parse(source: string | unknown) { + try { + return parseWorkflowDefinition(source) + } catch (error) { + throw new WorkflowDefinitionStoreError(error instanceof Error ? error.message : String(error), 400) + } + } + + private async readFile(id: string): Promise { + try { + const stored = JSON.parse(await fs.readFile(this.definitionPath(id), "utf8")) as StoredDefinition + if (stored.version !== 1 || stored.id !== id || !Number.isInteger(stored.currentRevision) + || stored.currentRevision < 1 || stored.currentRevision > WORKFLOW_DEFINITION_REVISION_LIMIT + || !Array.isArray(stored.revisions)) { + throw new Error(`Invalid stored workflow definition ${id}`) + } + if (stored.currentRevision !== stored.revisions.length || stored.revisions.length === 0) { + throw new Error(`Invalid stored workflow definition ${id}`) + } + for (const [index, record] of stored.revisions.entries()) { + if (record.id !== id || record.definition.id !== id || record.revision !== index + 1) throw new Error(`Invalid stored workflow definition ${id}`) + const parsed = parseWorkflowDefinition(record.definition) + if (record.canonical !== parsed.canonical) throw new Error(`Invalid canonical workflow definition ${id}`) + } + return stored + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + throw error + } + } + + private async persist(stored: StoredDefinition): Promise { + await fs.mkdir(this.directory, { recursive: true }) + const destination = this.definitionPath(stored.id) + const temporary = `${destination}.${randomUUID()}.tmp` + try { + await fs.writeFile(temporary, `${JSON.stringify(stored, null, 2)}\n`, { encoding: "utf8", flag: "wx" }) + await fs.rename(temporary, destination) + } catch (error) { + await fs.rm(temporary, { force: true }).catch(() => undefined) + throw error + } + } +} diff --git a/packages/server/src/workflows/interpreter.ts b/packages/server/src/workflows/interpreter.ts new file mode 100644 index 000000000..fdad2e35a --- /dev/null +++ b/packages/server/src/workflows/interpreter.ts @@ -0,0 +1,782 @@ +import { randomUUID } from "node:crypto" +import { isDeepStrictEqual } from "node:util" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { + WorkflowCondition, + WorkflowBudget, + WorkflowExecutionNode, + WorkflowNode, + WorkflowRun, + WorkflowUsage, + WorkflowValue, +} from "../api-types" +import { WORKFLOW_LIMITS } from "./definition-schema" +import { validateJsonSchemaValue } from "./json-schema" + +const DEFAULT_TIMEOUT_MS = 30 * 60 * 1_000 +const MAX_OUTPUT_CHARS = 16_000 +const MAX_RUN_OUTPUT_CHARS = 4_000_000 +const MAX_CONTEXT_BYTES = 256_000 +const MAX_CONTEXT_VALUES = 50_000 +const SAFE_AGENT_TOOL_IDS = new Set(["read", "glob", "grep", "lsp"]) + +export class WorkflowSuspendedError extends Error {} +export class WorkflowBudgetError extends Error {} +class WorkflowAmbiguousSideEffectError extends Error {} + +export interface WorkflowInterpreterOptions { + run: WorkflowRun + client: OpencodeClient + persist: () => Promise + signal: (timeoutMs: number) => AbortSignal + sessionStarted: (sessionId: string) => boolean + sessionFinished: (sessionId: string) => void + abortSession: (sessionId: string) => Promise + isCancelled: () => boolean + isPauseCommitted?: () => boolean +} + +interface ExecutionContext { + vars: Record + inputs: Record + budgets: WorkflowBudget[] + limiters: ActionLimiter[] + definitionInvocationKey: string + instanceKey?: string +} + +interface ActionLimiter { + max: number + active: number + waiters: Array<{ + resolve: () => void + reject: (reason?: unknown) => void + signal: AbortSignal + abort: () => void + }> +} + +interface ActionResult { + output: unknown + sessionId: string +} + +const emptyUsage = (): WorkflowUsage => ({ + cost: 0, + tokens: 0, + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}) + +export class WorkflowInterpreter { + private readonly run: WorkflowRun + private readonly nodes: WorkflowExecutionNode[] + private expanded = 0 + private outputChars = 0 + private readonly schedulerAbort = new AbortController() + private readonly maxConcurrency: number + private readonly rootLimiter: ActionLimiter + private readonly budgetLimiters = new Map() + private readonly savedDefinitionKeys: Set + + constructor(private readonly options: WorkflowInterpreterOptions) { + this.run = options.run + this.nodes = this.run.executionNodes ??= [] + this.run.usage ??= emptyUsage() + this.expanded = this.nodes.length + this.outputChars = this.nodes.reduce((total, node) => total + (node.output === undefined ? 0 : JSON.stringify(node.output).length), 0) + this.maxConcurrency = this.run.definitionSnapshot?.maxConcurrency ?? 1 + this.rootLimiter = { max: this.maxConcurrency, active: 0, waiters: [] } + this.savedDefinitionKeys = new Set((this.run.savedDefinitionSnapshots ?? []).map((snapshot) => `${snapshot.id}@${snapshot.revision}`)) + } + + async execute(): Promise { + const definition = this.run.definitionSnapshot + if (!definition) throw new Error("Workflow definition snapshot is missing") + if (!this.run.rootSessionId) { + const root = await this.requireData(this.options.client.session.create({ + ...(this.run.initiatorSessionId ? { parentID: this.run.initiatorSessionId } : {}), + title: `Workflow: ${this.run.objective.slice(0, 80)}`, + metadata: this.sessionMetadata("workflow"), + }, { signal: this.operationSignal(DEFAULT_TIMEOUT_MS) }), "create workflow session") + this.options.sessionStarted(root.id) + try { + this.throwIfCancelled() + this.run.rootSessionId = root.id + await this.options.persist() + this.options.sessionFinished(root.id) + } catch (error) { + if (!await this.options.abortSession(root.id)) { + throw new WorkflowAmbiguousSideEffectError(`Workflow root session abort was not confirmed: ${this.errorMessage(error)}`) + } + this.options.sessionFinished(root.id) + throw error + } + } + const definitionInvocationKey = `${this.run.definitionId}@${this.run.definitionRevision}` + await this.executeNode(definition.root, definition.root.id, { + vars: {}, + inputs: this.run.inputs ?? {}, + budgets: definition.budget ? [definition.budget] : [], + // ponytail: usage is observed after actions, so budgeted actions serialize until nodes declare reservable maxima. + limiters: [this.rootLimiter, ...(definition.budget ? [this.budgetLimiter(definitionInvocationKey)] : [])], + definitionInvocationKey, + }) + } + + private async executeNode(node: WorkflowNode, instanceKey: string, context: ExecutionContext, parentInstanceKey?: string): Promise { + this.throwIfCancelled() + const existing = this.nodes.find((candidate) => candidate.instanceKey === instanceKey) + if (existing?.status === "completed" || existing?.status === "skipped") return existing.output + await this.pauseIfRequested() + + const execution = existing ?? this.addExecution(node, instanceKey, context.definitionInvocationKey, parentInstanceKey) + const scopedContext = { ...context, instanceKey } + if (node.if !== undefined && !this.evaluateCondition(node.if, scopedContext)) { + execution.status = "skipped" + execution.completedAt = new Date().toISOString() + await this.options.persist() + return undefined + } + + execution.status = "running" + execution.startedAt ??= new Date().toISOString() + delete execution.error + await this.options.persist() + + let actionSessionId: string | undefined + try { + let output: unknown + switch (node.type) { + case "sequence": + output = await this.executeSequence(node.steps, instanceKey, scopedContext) + break + case "parallel": + output = await this.executeParallel(node.branches, instanceKey, scopedContext, node.maxConcurrency) + break + case "foreach": + output = await this.executeForeach(node, instanceKey, scopedContext) + break + case "repeat": + output = await this.executeRepeat(node, instanceKey, scopedContext) + break + case "condition": { + const selected = this.evaluateCondition(node.condition, scopedContext) ? node.then : node.else + output = selected ? await this.executeNode(selected, `${instanceKey}/${selected.id}`, scopedContext, instanceKey) : undefined + break + } + case "gate": + return await this.executeGate(node, execution) + case "agent": + ({ output, sessionId: actionSessionId } = await this.executeAgent(node, execution, scopedContext)) + break + case "shell": + ({ output, sessionId: actionSessionId } = await this.executeShell(node, execution, scopedContext)) + break + case "workflow": + output = await this.executeSavedWorkflow(node, instanceKey, scopedContext) + break + } + const structural = !["agent", "shell", "gate"].includes(node.type) + const bounded = this.boundOutput(output, structural) + if (bounded.truncated) throw new Error(`Workflow node ${node.id} output exceeds ${MAX_OUTPUT_CHARS} characters`) + const outputChars = output === undefined ? 0 : JSON.stringify(output).length + if (this.outputChars + outputChars > MAX_RUN_OUTPUT_CHARS) throw new Error("Workflow run output limit exceeded") + this.outputChars += outputChars + execution.output = bounded.output + execution.status = "completed" + execution.completedAt = new Date().toISOString() + await this.options.persist() + if (actionSessionId) this.options.sessionFinished(actionSessionId) + return output + } catch (caught) { + let error = caught + if (actionSessionId) { + if (await this.options.abortSession(actionSessionId)) { + this.options.sessionFinished(actionSessionId) + this.forgetSession(execution, actionSessionId) + } + else error = new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + } + if (error instanceof WorkflowSuspendedError) { + execution.status = "waiting" + await this.options.persist() + } else if (error instanceof WorkflowAmbiguousSideEffectError) { + execution.status = "interrupted" + execution.error = this.errorMessage(error) + execution.completedAt = new Date().toISOString() + await this.options.persist() + } else if (!this.options.isCancelled()) { + execution.status = "failed" + execution.error = this.errorMessage(error) + execution.completedAt = new Date().toISOString() + await this.options.persist() + } + throw error + } + } + + private async executeSequence(steps: WorkflowNode[], parent: string, context: ExecutionContext): Promise { + let output: unknown + for (const child of steps) output = await this.executeNode(child, `${parent}/${child.id}`, context, parent) + return output + } + + private async executeParallel(branches: WorkflowNode[], parent: string, context: ExecutionContext, concurrency?: number): Promise { + return this.mapBounded(branches, concurrency, (branch) => + this.executeNode(branch, `${parent}/${branch.id}`, { + vars: { ...context.vars }, inputs: context.inputs, budgets: context.budgets, limiters: context.limiters, + definitionInvocationKey: context.definitionInvocationKey, + }, parent)) + } + + private async executeForeach(node: Extract, parent: string, context: ExecutionContext): Promise { + const value = this.resolveValue(node.items, context) + if (!Array.isArray(value)) throw new Error(`Foreach node ${node.id} items must resolve to an array`) + if (value.length > node.maxItems) throw new Error(`Foreach node ${node.id} exceeded maxItems ${node.maxItems}`) + return this.mapBounded(value, node.maxConcurrency, (item, index) => this.executeNode( + node.body, + `${parent}/${node.body.id}[${index}]`, + { + vars: { ...context.vars, [node.item]: item, [`${node.item}Index`]: index }, + inputs: context.inputs, budgets: context.budgets, limiters: context.limiters, + definitionInvocationKey: context.definitionInvocationKey, + }, + parent, + )) + } + + private async executeRepeat(node: Extract, parent: string, context: ExecutionContext): Promise { + const output: unknown[] = [] + for (let index = 0; index < node.maxIterations; index += 1) { + const iterationContext = { + vars: { ...context.vars, iteration: index }, inputs: context.inputs, budgets: context.budgets, limiters: context.limiters, + definitionInvocationKey: context.definitionInvocationKey, instanceKey: context.instanceKey, + } + if (node.while !== undefined && !this.evaluateCondition(node.while, iterationContext)) break + output.push(await this.executeNode(node.body, `${parent}/${node.body.id}[${index}]`, iterationContext, parent)) + } + return output + } + + private async executeGate(node: Extract, execution: WorkflowExecutionNode): Promise { + if (execution.status === "completed") return execution.output + if (this.run.pendingGate && this.run.pendingGate.executionNodeId !== execution.id) throw new WorkflowSuspendedError() + execution.status = "waiting" + this.run.pendingGate = { + executionNodeId: execution.id, + definitionNodeId: node.id, + gate: node.gate, + prompt: node.prompt, + ...(node.inputSchema ? { inputSchema: node.inputSchema } : {}), + } + this.run.status = node.gate === "approval" ? "waiting_for_review" : "waiting_for_input" + await this.options.persist() + throw new WorkflowSuspendedError() + } + + private async executeAgent( + node: Extract, + execution: WorkflowExecutionNode, + context: ExecutionContext, + ): Promise { + const signal = this.operationSignal(node.timeoutMs ?? DEFAULT_TIMEOUT_MS) + const contextValue = node.context === undefined ? undefined : this.resolveContext(node.context, context, signal) + const prompt = [ + `Workflow node: ${node.title ?? node.id}`, + "", + `Objective:\n${this.run.objective}`, + "", + `Instructions:\n${node.instructions}`, + ...(contextValue === undefined ? [] : ["", `Context:\n${JSON.stringify(contextValue, null, 2)}`]), + ].join("\n") + return this.withActionPermit(context.limiters, signal, async () => { + this.enforceActionAdmission(context.budgets) + const tools = await this.toolOverrides(node.tools ?? [], signal) + return this.retry(node.retry?.maxAttempts ?? 1, node.retry?.delayMs ?? 0, execution, context.budgets, signal, async () => { + const sessionId = await this.createChildSession(node, execution, signal) + try { + const response = await this.requireData(this.options.client.session.prompt({ + sessionID: sessionId, + ...(node.agent ? { agent: node.agent } : {}), + ...(node.model ? { model: node.model } : {}), + tools, + ...(node.outputSchema ? { format: { type: "json_schema" as const, schema: node.outputSchema, retryCount: 2 } } : {}), + parts: [{ type: "text", text: prompt }], + }, { signal }), `run ${node.id} session`) + this.observeUsage(response.info, execution, context.budgets) + if (response.info.error) throw new Error(this.errorMessage(response.info.error)) + if (node.outputSchema) { + if (response.info.structured === undefined) throw new Error(`Structured output is missing for ${node.id}`) + const issues = validateJsonSchemaValue(response.info.structured, node.outputSchema) + if (issues.length) throw new Error(`Structured output is invalid for ${node.id}: ${issues.join("; ")}`) + return { output: response.info.structured, sessionId } + } + return { output: response.parts.filter((part) => part.type === "text").map((part) => part.text).join("\n"), sessionId } + } catch (error) { + if (await this.options.abortSession(sessionId)) { + this.options.sessionFinished(sessionId) + this.forgetSession(execution, sessionId) + } + else throw new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + throw error + } + }) + }) + } + + private async executeShell( + node: Extract, + execution: WorkflowExecutionNode, + context: ExecutionContext, + ): Promise { + const signal = this.operationSignal(node.timeoutMs ?? DEFAULT_TIMEOUT_MS) + return this.withActionPermit(context.limiters, signal, () => this.retry( + node.retry?.maxAttempts ?? 1, node.retry?.delayMs ?? 0, execution, context.budgets, signal, async () => { + this.enforceActionAdmission(context.budgets) + const sessionId = await this.createChildSession(node, execution, signal) + try { + const response = await this.requireData(this.options.client.session.shell({ + sessionID: sessionId, + agent: node.agent, + command: node.command, + ...(node.model ? { model: node.model } : {}), + }, { signal }), `run ${node.id} shell`) + this.observeUsage(response.info, execution, context.budgets) + if ("error" in response.info && response.info.error) throw new Error(this.errorMessage(response.info.error)) + const toolParts = response.parts.filter((part) => part.type === "tool") + const toolError = toolParts.find((part) => part.state.status === "error") + if (toolError?.state.status === "error") throw new Error(toolError.state.error) + const output = toolParts.filter((part) => part.state.status === "completed") + .map((part) => part.state.status === "completed" ? part.state.output : "").join("\n") + return { output: output || response.parts.filter((part) => part.type === "text").map((part) => part.text).join("\n"), sessionId } + } catch (error) { + if (await this.options.abortSession(sessionId)) { + this.options.sessionFinished(sessionId) + this.forgetSession(execution, sessionId) + } + else throw new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + throw error + } + })) + } + + private async executeSavedWorkflow( + node: Extract, + instanceKey: string, + context: ExecutionContext, + ): Promise { + const snapshot = this.run.savedDefinitionSnapshots?.find((candidate) => + candidate.id === node.definitionId && candidate.revision === node.definitionRevision) + if (!snapshot) throw new Error(`Saved workflow snapshot ${node.definitionId}@${node.definitionRevision ?? "unresolved"} is missing`) + const inputs = Object.fromEntries(Object.entries(node.inputs ?? {}).map(([key, value]) => [key, this.resolveValue(value, context)])) + const root = snapshot.definition.root + const definitionInvocationKey = `${instanceKey}/${snapshot.id}@${snapshot.revision}` + return this.executeNode(root, `${definitionInvocationKey}/${root.id}`, { + vars: {}, + inputs, + budgets: snapshot.definition.budget ? [...context.budgets, snapshot.definition.budget] : context.budgets, + limiters: [ + ...context.limiters, + this.actionLimiter(snapshot.definition.maxConcurrency ?? 1), + ...(snapshot.definition.budget ? [this.budgetLimiter(`${snapshot.id}@${snapshot.revision}`)] : []), + ], + definitionInvocationKey, + }, instanceKey) + } + + private async createChildSession( + node: Extract, + execution: WorkflowExecutionNode, + signal: AbortSignal, + ): Promise { + this.throwIfCancelled() + const session = await this.requireData(this.options.client.session.create({ + parentID: this.run.rootSessionId, + title: `${node.title ?? node.id}: ${this.run.objective.slice(0, 60)}`, + ...(node.agent ? { agent: node.agent } : {}), + metadata: this.sessionMetadata(node.id), + }, { signal }), `create ${node.id} session`) + execution.sessionIds ??= [] + execution.sessionIds.push(session.id) + const accepted = this.options.sessionStarted(session.id) + try { + if (!accepted) this.throwIfCancelled() + signal.throwIfAborted() + await this.options.persist() + signal.throwIfAborted() + } catch (error) { + if (await this.options.abortSession(session.id)) { + this.options.sessionFinished(session.id) + this.forgetSession(execution, session.id) + } + else throw new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + throw error + } + return session.id + } + + private async retry( + attempts: number, + delayMs: number, + execution: WorkflowExecutionNode, + budgets: WorkflowBudget[], + signal: AbortSignal, + operation: () => Promise, + ): Promise { + let lastError: unknown = new Error("Workflow retry limit was already exhausted") + for (let attempt = execution.attempt + 1; attempt <= attempts; attempt += 1) { + signal.throwIfAborted() + this.enforceActionAdmission(budgets) + execution.attempt = attempt + await this.options.persist() + signal.throwIfAborted() + try { + return await operation() + } catch (error) { + lastError = error + if (error instanceof WorkflowAmbiguousSideEffectError || error instanceof WorkflowBudgetError) throw error + this.throwIfCancelled() + signal.throwIfAborted() + this.enforceActionAdmission(budgets) + if (attempt < attempts && delayMs) await this.sleep(delayMs, signal) + } + } + throw lastError + } + + private async mapBounded(items: T[], requested: number | undefined, operation: (item: T, index: number) => Promise): Promise { + const concurrency = Math.min(requested ?? this.maxConcurrency, this.maxConcurrency, WORKFLOW_LIMITS.concurrency, items.length || 1) + const results = new Array(items.length) + let next = 0 + let suspended: WorkflowSuspendedError | undefined + let failed: unknown + const workers = Array.from({ length: concurrency }, async () => { + while (next < items.length && !suspended && failed === undefined) { + const index = next++ + try { + results[index] = await operation(items[index]!, index) + } catch (error) { + if (error instanceof WorkflowSuspendedError) suspended = error + else if (failed === undefined) { failed = error; this.schedulerAbort.abort(error) } + } + } + }) + await Promise.all(workers) + if (failed !== undefined) throw failed + if (suspended) throw suspended + return results + } + + private addExecution( + node: WorkflowNode, + instanceKey: string, + definitionInvocationKey: string, + parentInstanceKey?: string, + ): WorkflowExecutionNode { + const limit = this.run.definitionSnapshot?.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + if (++this.expanded > limit) throw new Error(`Workflow exceeded expanded node limit ${limit}`) + const execution: WorkflowExecutionNode = { + id: randomUUID(), + instanceKey, + definitionInvocationKey, + definitionNodeId: node.id, + type: node.type, + status: "pending", + attempt: 0, + ...(parentInstanceKey ? { parentInstanceKey } : {}), + } + this.nodes.push(execution) + return execution + } + + private resolveValue(value: WorkflowValue, context: ExecutionContext, visit?: () => void): unknown { + visit?.() + if (Array.isArray(value)) return value.map((item) => this.resolveValue(item, context, visit)) + if (!value || typeof value !== "object") return value + if (Object.prototype.hasOwnProperty.call(value, "$ref") && typeof value.$ref === "string") return this.resolveRef(value.$ref, context) + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, context, visit)])) + } + + private resolveContext(value: WorkflowValue, context: ExecutionContext, signal: AbortSignal): unknown { + const resolved = this.resolveValue(value, context, () => signal.throwIfAborted()) + const pending: Array<{ value: unknown; exit?: boolean }> = [{ value: resolved }] + const ancestors = new WeakSet() + let count = 0 + while (pending.length) { + signal.throwIfAborted() + const { value: current, exit } = pending.pop()! + if (exit) { + ancestors.delete(current as object) + continue + } + if (++count > MAX_CONTEXT_VALUES) throw new Error(`Workflow context exceeds ${MAX_CONTEXT_VALUES} values`) + if (!current || typeof current !== "object") continue + if (ancestors.has(current)) throw new Error("Workflow context contains a cycle") + ancestors.add(current) + pending.push({ value: current, exit: true }) + pending.push(...Object.values(current).reverse().map((value) => ({ value }))) + } + const serialized = JSON.stringify(resolved) + if (serialized !== undefined && Buffer.byteLength(serialized, "utf8") > MAX_CONTEXT_BYTES) { + throw new Error(`Workflow context exceeds ${MAX_CONTEXT_BYTES} bytes`) + } + return resolved + } + + private resolveRef(ref: string, context: ExecutionContext): unknown { + const [root, name, ...path] = ref.split(".") + let value: unknown + if (root === "inputs" && Object.prototype.hasOwnProperty.call(context.inputs, name)) value = context.inputs[name!] + else if (root === "vars" && Object.prototype.hasOwnProperty.call(context.vars, name)) value = context.vars[name!] + else if (root === "nodes") { + const candidates = this.nodes.filter((node) => node.definitionNodeId === name && node.status === "completed" + && this.executionDefinitionInvocationKey(node) === context.definitionInvocationKey).reverse() + value = candidates.sort((left, right) => this.commonPrefix(right.instanceKey, context.instanceKey ?? "") + - this.commonPrefix(left.instanceKey, context.instanceKey ?? "")).at(0) + } + for (const part of path) { + if (!value || typeof value !== "object" || !Object.prototype.hasOwnProperty.call(value, part)) return undefined + value = (value as Record)[part] + } + return value + } + + private evaluateCondition(condition: WorkflowCondition, context: ExecutionContext): boolean { + if (typeof condition === "boolean") return condition + const actual = this.resolveValue(condition.value, context) + if (condition.equals !== undefined) return this.equal(actual, this.resolveValue(condition.equals, context)) + if (condition.notEquals !== undefined) return !this.equal(actual, this.resolveValue(condition.notEquals, context)) + if (condition.exists !== undefined) return (actual !== undefined) === condition.exists + return Boolean(actual) === (condition.truthy ?? true) + } + + private equal(left: unknown, right: unknown): boolean { + return isDeepStrictEqual(left, right) + } + + private commonPrefix(left: string, right: string): number { + let index = 0 + while (index < left.length && left[index] === right[index]) index += 1 + return index + } + + private observeUsage(info: unknown, execution: WorkflowExecutionNode, budgets: WorkflowBudget[]) { + if (!info || typeof info !== "object" + || !Object.prototype.hasOwnProperty.call(info, "cost") + || !Object.prototype.hasOwnProperty.call(info, "tokens")) return + const message = info as { cost?: number; tokens?: { total?: number; input?: number; output?: number; reasoning?: number; cache?: { read?: number; write?: number } } } + const number = (value: unknown, field: string) => { + if (value === undefined) return 0 + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Workflow SDK usage ${field} must be finite and non-negative`) + } + return value + } + if (!message.tokens || typeof message.tokens !== "object") throw new Error("Workflow SDK usage tokens must be an object") + const inputTokens = number(message.tokens.input, "tokens.input") + const outputTokens = number(message.tokens.output, "tokens.output") + const reasoningTokens = number(message.tokens.reasoning, "tokens.reasoning") + const cacheReadTokens = number(message.tokens.cache?.read, "tokens.cache.read") + const cacheWriteTokens = number(message.tokens.cache?.write, "tokens.cache.write") + const add = (left: number, right: number, field: string) => { + if (!Number.isFinite(left) || left < 0 || !Number.isFinite(right) || right < 0) { + throw new Error(`Workflow usage ${field} must be finite and non-negative`) + } + const total = left + right + if (!Number.isFinite(total)) throw new Error(`Workflow usage ${field} overflowed`) + return total + } + const usage: WorkflowUsage = { + cost: number(message.cost, "cost"), + tokens: message.tokens.total === undefined + ? [outputTokens, reasoningTokens, cacheReadTokens, cacheWriteTokens] + .reduce((total, value) => add(total, value, "tokens.total"), inputTokens) + : number(message.tokens.total, "tokens.total"), + inputTokens, + outputTokens, + reasoningTokens, + cacheReadTokens, + cacheWriteTokens, + } + const nextUsage = emptyUsage() + for (const key of Object.keys(usage) as Array) { + nextUsage[key] = add(this.run.usage![key], usage[key], key) + } + execution.usage = usage + this.run.usage = nextUsage + try { + this.enforceObservedBudget(budgets) + } catch (error) { + this.schedulerAbort.abort(error) + throw error + } + } + + private async toolOverrides(allowed: string[], signal: AbortSignal): Promise> { + for (const id of allowed) { + if (!SAFE_AGENT_TOOL_IDS.has(id)) throw new Error(`Workflow agent tool ${id} is not allowed`) + } + const ids = await this.requireData( + this.options.client.tool.ids(undefined, { signal }), + "list workflow tools", + ) + const installed = new Set(ids) + const overrides: Record = { "*": false, ...Object.fromEntries(ids.map((id) => [id, false])) } + for (const id of allowed) { + if (!installed.has(id)) throw new Error(`Workflow agent tool ${id} is unavailable`) + overrides[id] = true + } + return overrides + } + + private operationSignal(timeoutMs: number): AbortSignal { + return AbortSignal.any([this.options.signal(timeoutMs), this.schedulerAbort.signal]) + } + + private async withActionPermit(limiters: ActionLimiter[], signal: AbortSignal, operation: () => Promise): Promise { + const acquired: ActionLimiter[] = [] + try { + for (const limiter of limiters) { + await this.acquirePermit(limiter, signal) + acquired.push(limiter) + signal.throwIfAborted() + this.throwIfCancelled() + await this.pauseIfRequested() + } + return await operation() + } finally { + for (const limiter of acquired.reverse()) this.releasePermit(limiter) + } + } + + private async acquirePermit(limiter: ActionLimiter, signal: AbortSignal): Promise { + signal.throwIfAborted() + if (limiter.active < limiter.max && limiter.waiters.length === 0) { + limiter.active += 1 + return + } + await new Promise((resolve, reject) => { + const waiter = { resolve, reject, signal, abort: () => undefined as void } + waiter.abort = () => { + const index = limiter.waiters.indexOf(waiter) + if (index >= 0) limiter.waiters.splice(index, 1) + reject(signal.reason) + } + limiter.waiters.push(waiter) + signal.addEventListener("abort", waiter.abort, { once: true }) + if (signal.aborted) waiter.abort() + }) + } + + private releasePermit(limiter: ActionLimiter): void { + while (limiter.waiters.length) { + const waiter = limiter.waiters.shift()! + waiter.signal.removeEventListener("abort", waiter.abort) + if (waiter.signal.aborted) { + waiter.reject(waiter.signal.reason) + continue + } + waiter.resolve() + return + } + limiter.active -= 1 + } + + private async sleep(delayMs: number, signal: AbortSignal): Promise { + await new Promise((resolve, reject) => { + const abort = () => { clearTimeout(timer); reject(signal.reason) } + const timer = setTimeout(() => { signal.removeEventListener("abort", abort); resolve() }, delayMs) + signal.addEventListener("abort", abort, { once: true }) + }) + } + + private actionLimiter(max: number): ActionLimiter { + return { max, active: 0, waiters: [] } + } + + private budgetLimiter(key: string): ActionLimiter { + const existing = this.budgetLimiters.get(key) + if (existing) return existing + const limiter = this.actionLimiter(1) + this.budgetLimiters.set(key, limiter) + return limiter + } + + private executionDefinitionInvocationKey(execution: WorkflowExecutionNode): string { + if (execution.definitionInvocationKey) return execution.definitionInvocationKey + const segments = execution.instanceKey.split("/") + const saved = segments.map((segment, index) => ({ segment, index })) + .filter(({ segment }) => this.savedDefinitionKeys.has(segment)).at(-1) + return saved ? segments.slice(0, saved.index + 1).join("/") : `${this.run.definitionId}@${this.run.definitionRevision}` + } + + private forgetSession(execution: WorkflowExecutionNode, sessionId: string): void { + if (!execution.sessionIds) return + execution.sessionIds = execution.sessionIds.filter((candidate) => candidate !== sessionId) + if (execution.sessionIds.length === 0) delete execution.sessionIds + } + + private enforceActionAdmission(budgets: WorkflowBudget[]) { + for (const budget of budgets) { + if (budget.maxCost !== undefined && this.run.usage!.cost >= budget.maxCost) { + throw new WorkflowBudgetError(`Workflow reached cost budget ${budget.maxCost}`) + } + if (budget.maxTokens !== undefined && this.run.usage!.tokens >= budget.maxTokens) { + throw new WorkflowBudgetError(`Workflow reached token budget ${budget.maxTokens}`) + } + } + } + + private enforceObservedBudget(budgets: WorkflowBudget[]) { + // ponytail: providers report usage after an action, so one admitted action can overshoot; no estimate is invented. + for (const budget of budgets) { + if (budget.maxCost !== undefined && this.run.usage!.cost > budget.maxCost) { + throw new WorkflowBudgetError(`Workflow exceeded cost budget ${budget.maxCost}`) + } + if (budget.maxTokens !== undefined && this.run.usage!.tokens > budget.maxTokens) { + throw new WorkflowBudgetError(`Workflow exceeded token budget ${budget.maxTokens}`) + } + } + } + + private async pauseIfRequested() { + if (!this.run.pauseRequested || this.options.isPauseCommitted?.() === false) return + throw new WorkflowSuspendedError() + } + + private throwIfCancelled() { + if (this.options.isCancelled()) throw new Error("Workflow run cancelled") + } + + private boundOutput(output: unknown, structural = false): { output: unknown; truncated: boolean } { + if (output === undefined) return { output: undefined, truncated: false } + if (structural) return { output, truncated: false } + if (typeof output === "string") return output.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: output.slice(0, MAX_OUTPUT_CHARS), truncated: true } + const serialized = JSON.stringify(output) + return serialized.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: serialized.slice(0, MAX_OUTPUT_CHARS), truncated: true } + } + + private sessionMetadata(role: string) { + return { codenomad: { version: 1, workflow: { runId: this.run.id, role } } } + } + + private async requireData(request: Promise<{ data?: T; error?: unknown }>, action: string): Promise { + const response = await request + if (response.data !== undefined) return response.data + throw new Error(`${action} failed: ${this.errorMessage(response.error)}`) + } + + private errorMessage(error: unknown): string { + if (error instanceof Error) return error.message + if (typeof error === "string") return error + try { return JSON.stringify(error) || "Unknown error" } catch { return "Unknown error" } + } +} diff --git a/packages/server/src/workflows/json-schema.ts b/packages/server/src/workflows/json-schema.ts new file mode 100644 index 000000000..d2d0c9225 --- /dev/null +++ b/packages/server/src/workflows/json-schema.ts @@ -0,0 +1,168 @@ +const JSON_TYPES = new Set(["null", "array", "object", "integer", "number", "string", "boolean"]) +const SCHEMA_KEYS = new Set([ + "type", "enum", "const", "minLength", "maxLength", "minimum", "maximum", + "minItems", "maxItems", "items", "required", "properties", "additionalProperties", + "allOf", "anyOf", "oneOf", +]) +const SCHEMA_DEPTH_LIMIT = 20 +const SCHEMA_NODE_LIMIT = 2_000 + +export interface JsonSchemaIssue { + path: Array + message: string +} + +const isRecord = (value: unknown): value is Record => + Boolean(value) && typeof value === "object" && !Array.isArray(value) + +const nonNegativeInteger = (value: unknown) => Number.isInteger(value) && (value as number) >= 0 + +const jsonEqual = (left: unknown, right: unknown): boolean => { + if (left === right) return true + if (Array.isArray(left) || Array.isArray(right)) { + return Array.isArray(left) && Array.isArray(right) && left.length === right.length + && left.every((value, index) => jsonEqual(value, right[index])) + } + if (!isRecord(left) || !isRecord(right)) return false + const keys = Object.keys(left) + return keys.length === Object.keys(right).length + && keys.every((key) => Object.prototype.hasOwnProperty.call(right, key) && jsonEqual(left[key], right[key])) +} + +export function inspectJsonSchema(schema: Record): JsonSchemaIssue[] { + const issues: JsonSchemaIssue[] = [] + const pending: Array<{ schema: Record; path: Array; depth: number }> = [ + { schema, path: [], depth: 0 }, + ] + let nodes = 0 + + while (pending.length) { + const current = pending.pop()! + if (++nodes > SCHEMA_NODE_LIMIT) { + issues.push({ path: current.path, message: "JSON schema has too many nested schemas" }) + break + } + if (current.depth > SCHEMA_DEPTH_LIMIT) { + issues.push({ path: current.path, message: "JSON schema is too deeply nested" }) + continue + } + const add = (key: string, message: string) => issues.push({ path: [...current.path, key], message }) + + for (const key of Object.keys(current.schema)) { + if (!SCHEMA_KEYS.has(key)) add(key, `Unsupported JSON schema keyword: ${key}`) + } + + if (current.schema.type !== undefined) { + const types = typeof current.schema.type === "string" ? [current.schema.type] : current.schema.type + if (!Array.isArray(types) || types.length === 0 || types.some((type) => typeof type !== "string" || !JSON_TYPES.has(type))) { + add("type", "JSON schema type must contain only supported JSON types") + } else if (new Set(types).size !== types.length) { + add("type", "JSON schema types must be unique") + } + } + if (current.schema.enum !== undefined) { + if (!Array.isArray(current.schema.enum)) add("enum", "JSON schema enum must be an array") + else if (current.schema.enum.length === 0) add("enum", "JSON schema enum must not be empty") + else if (current.schema.enum.some((value, index, values) => values.slice(0, index).some((other) => jsonEqual(value, other)))) { + add("enum", "JSON schema enum values must be unique") + } + } + for (const key of ["minLength", "maxLength", "minItems", "maxItems"] as const) { + if (current.schema[key] !== undefined && !nonNegativeInteger(current.schema[key])) add(key, `${key} must be a non-negative integer`) + } + for (const key of ["minimum", "maximum"] as const) { + if (current.schema[key] !== undefined && (typeof current.schema[key] !== "number" || !Number.isFinite(current.schema[key]))) { + add(key, `${key} must be a finite number`) + } + } + if (current.schema.required !== undefined) { + const required = current.schema.required + if (!Array.isArray(required) || required.some((key) => typeof key !== "string")) add("required", "required must be an array of property names") + else if (new Set(required).size !== required.length) add("required", "required property names must be unique") + } + if (current.schema.additionalProperties !== undefined && typeof current.schema.additionalProperties !== "boolean") { + add("additionalProperties", "additionalProperties must be a boolean") + } + + const enqueue = (value: unknown, path: Array) => { + if (!isRecord(value)) { + issues.push({ path, message: "Nested JSON schema must be an object" }) + return + } + pending.push({ schema: value, path, depth: current.depth + 1 }) + } + if (current.schema.items !== undefined) enqueue(current.schema.items, [...current.path, "items"]) + if (current.schema.properties !== undefined) { + if (!isRecord(current.schema.properties)) add("properties", "properties must be an object") + else for (const [key, child] of Object.entries(current.schema.properties)) enqueue(child, [...current.path, "properties", key]) + } + for (const key of ["allOf", "anyOf", "oneOf"] as const) { + const alternatives = current.schema[key] + if (alternatives === undefined) continue + if (!Array.isArray(alternatives) || alternatives.length === 0) { + add(key, `${key} must be a non-empty array of schemas`) + continue + } + alternatives.forEach((child, index) => enqueue(child, [...current.path, key, index])) + } + } + return issues +} + +const typeMatches = (value: unknown, type: string) => { + if (type === "null") return value === null + if (type === "array") return Array.isArray(value) + if (type === "object") return Boolean(value) && typeof value === "object" && !Array.isArray(value) + if (type === "integer") return Number.isInteger(value) + if (type === "number") return typeof value === "number" && Number.isFinite(value) + return typeof value === type +} + +export function validateJsonSchemaValue(value: unknown, schema: Record, path = "$"): string[] { + const errors: string[] = [] + const types = typeof schema.type === "string" ? [schema.type] : Array.isArray(schema.type) ? schema.type : [] + if (types.length && !types.some((type) => typeof type === "string" && typeMatches(value, type))) { + return [`${path} must be ${types.join(" or ")}`] + } + if (Array.isArray(schema.enum) && !schema.enum.some((candidate) => jsonEqual(candidate, value))) errors.push(`${path} is not an allowed value`) + if (Object.prototype.hasOwnProperty.call(schema, "const") && !jsonEqual(schema.const, value)) errors.push(`${path} must equal const`) + + if (typeof value === "string") { + const length = [...value].length + if (typeof schema.minLength === "number" && length < schema.minLength) errors.push(`${path} is too short`) + if (typeof schema.maxLength === "number" && length > schema.maxLength) errors.push(`${path} is too long`) + } + if (typeof value === "number") { + if (typeof schema.minimum === "number" && value < schema.minimum) errors.push(`${path} is below minimum`) + if (typeof schema.maximum === "number" && value > schema.maximum) errors.push(`${path} is above maximum`) + } + if (Array.isArray(value)) { + if (typeof schema.minItems === "number" && value.length < schema.minItems) errors.push(`${path} has too few items`) + if (typeof schema.maxItems === "number" && value.length > schema.maxItems) errors.push(`${path} has too many items`) + if (isRecord(schema.items)) value.forEach((item, index) => errors.push(...validateJsonSchemaValue(item, schema.items as Record, `${path}[${index}]`))) + } + if (isRecord(value)) { + if (Array.isArray(schema.required)) for (const key of schema.required) { + if (typeof key === "string" && !Object.prototype.hasOwnProperty.call(value, key)) errors.push(`${path}.${key} is required`) + } + const properties = isRecord(schema.properties) ? schema.properties : {} + for (const [key, childSchema] of Object.entries(properties)) { + if (Object.prototype.hasOwnProperty.call(value, key) && isRecord(childSchema)) { + errors.push(...validateJsonSchemaValue(value[key], childSchema, `${path}.${key}`)) + } + } + if (schema.additionalProperties === false) { + for (const key of Object.keys(value)) if (!Object.prototype.hasOwnProperty.call(properties, key)) errors.push(`${path}.${key} is not allowed`) + } + } + + for (const keyword of ["allOf", "anyOf", "oneOf"] as const) { + if (!Array.isArray(schema[keyword])) continue + const results = schema[keyword].filter(isRecord).map((item) => validateJsonSchemaValue(value, item, path)) + const matches = results.filter((result) => result.length === 0).length + if (keyword === "allOf" && matches !== results.length) errors.push(`${path} does not match allOf`) + if (keyword === "anyOf" && matches === 0) errors.push(`${path} does not match anyOf`) + if (keyword === "oneOf" && matches !== 1) errors.push(`${path} does not match exactly one oneOf schema`) + } + return errors +} diff --git a/packages/server/src/workflows/manager.test.ts b/packages/server/src/workflows/manager.test.ts new file mode 100644 index 000000000..9206a59c5 --- /dev/null +++ b/packages/server/src/workflows/manager.test.ts @@ -0,0 +1,397 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { EventBus } from "../events/bus" +import type { Logger } from "../logger" +import type { WorkspaceManager } from "../workspaces/manager" +import { WorkflowManager } from "./manager" + +describe("WorkflowManager", () => { + it("rejects a stale saved definition inside the admission queue", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-latest-")) + const manager = new WorkflowManager({ + workspaceManager: { list: () => [] } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + createClient: () => null, + }) + try { + const definition = { version: 1 as const, id: "latest", name: "Latest", root: { + type: "agent" as const, id: "work", instructions: "Work", + } } + await manager.createDefinition(definition) + await manager.listDefinitions() + const [updated, stale] = await Promise.allSettled([ + manager.updateDefinition("latest", 1, { ...definition, name: "Updated" }), + manager.startLatest({ workspaceId: "workspace", definitionId: "latest", definitionRevision: 1 }), + ]) + assert.equal(updated.status, "fulfilled") + assert.equal(stale.status, "rejected") + assert.match((stale as PromiseRejectedResult).reason.message, /revision is stale/) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("resolves an omitted latest revision inside admission", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-current-")) + let sessions = 0 + const client = { session: { + create: async () => ({ data: { id: `latest-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ + workspaceManager: { + get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + createClient: () => client, + }) + try { + const definition = { version: 1 as const, id: "current", name: "Current", root: { + type: "gate" as const, id: "gate", gate: "approval" as const, prompt: "Wait", + } } + await manager.createDefinition(definition) + const [updated, started] = await Promise.all([ + manager.updateDefinition("current", 1, { ...definition, name: "Updated" }), + manager.startLatest({ workspaceId: "workspace", definitionId: "current" }), + ]) + assert.equal(updated.revision, 2) + assert.equal(started.definitionRevision, 2) + assert.equal(started.definitionSnapshot?.name, "Updated") + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("retries transient creation cleanup on later admission", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-cleanup-")) + let attempts = 0 + const manager = new WorkflowManager({ + workspaceManager: { + list: () => [], + cancelCreationRequest: async () => { + if (++attempts === 1) throw new Error("transient cleanup failure") + }, + } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + try { + ;(manager as any).deferCreationCleanup("request") + await (manager as any).drainCreationCleanups() + assert.equal(attempts, 1) + assert.equal((manager as any).deferredCreationCleanups.has("request"), true) + await assert.rejects(manager.startLatest({ workspaceId: "workspace", definitionId: "missing" }), /not found/) + assert.equal(attempts, 2) + assert.equal((manager as any).deferredCreationCleanups.size, 0) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("persists and hands a structured planner result to the implementer", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflows-")) + const creates: Array | undefined> = [] + const prompts: Array> = [] + let session = 0 + const client = { + session: { + create: async (input?: Record) => { + creates.push(input) + return { data: { id: `session-${++session}` } } + }, + prompt: async (input: Record) => { + prompts.push(input) + if (prompts.length === 1) { + return { + data: { + info: { structured: { summary: "Plan", steps: ["Change code", "Run test"] } }, + parts: [], + }, + } + } + const text = prompts.length === 2 ? "Reviewed plan" : "Implemented" + return { data: { info: {}, parts: [{ type: "text", text }] } } + }, + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage-a", path: "C:/workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "lineage-a", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + const events: unknown[] = [] + const eventBus = { publish: (event: unknown) => { events.push(event); return true } } as unknown as EventBus + const logger = { warn() {}, error() {} } as unknown as Logger + const manager = new WorkflowManager({ + workspaceManager, + eventBus, + logger, + storageDir, + createClient: () => client, + }) + let reloaded: WorkflowManager | undefined + + try { + const started = await manager.start({ + workspaceId: "workspace", + objective: "Add workflow support", + stages: [ + { id: "planner", title: "Planner", instructions: "Create a plan", requiresApproval: true }, + { id: "reviewer", title: "Reviewer", instructions: "Review the approved plan", requiresApproval: true }, + { id: "implementer", title: "Implementer", instructions: "Implement the reviewed plan", requiresApproval: true }, + ], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "waiting_for_review") + assert.equal(run.rootSessionId, "session-1") + assert.deepEqual(run.steps.map((step) => [step.id, step.status, step.sessionId]), [ + ["planner", "completed", "session-2"], + ["reviewer", "pending", undefined], + ["implementer", "pending", undefined], + ]) + + run = (await manager.approve(started.id, "planner"))! + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "waiting_for_review") + assert.deepEqual(run.steps.map((step) => [step.id, step.status, step.sessionId]), [ + ["planner", "completed", "session-2"], + ["reviewer", "completed", "session-3"], + ["implementer", "pending", undefined], + ]) + + run = (await manager.approve(started.id, "reviewer"))! + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "waiting_for_review") + assert.deepEqual(run.steps.map((step) => [step.id, step.status, step.sessionId]), [ + ["planner", "completed", "session-2"], + ["reviewer", "completed", "session-3"], + ["implementer", "completed", "session-4"], + ]) + assert.equal(creates[1]?.parentID, "session-1") + assert.equal(creates[2]?.parentID, "session-1") + assert.match(JSON.stringify(prompts[1]), /Change code/) + assert.match(JSON.stringify(prompts[2]), /Reviewed plan/) + assert.ok(events.length >= 10) + + await manager.shutdown() + const restoredWorkspaceManager = { + get: (id: string) => id === "workspace-restored" + ? { id, lineageId: "lineage-a", path: "C:/workspace", status: "ready" } + : undefined, + list: () => [{ id: "workspace-restored", lineageId: "lineage-a", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + reloaded = new WorkflowManager({ + workspaceManager: restoredWorkspaceManager, + eventBus, + logger, + storageDir, + createClient: () => client, + }) + await assert.rejects( + reloaded.start({ + workspaceId: "workspace-restored", + objective: "Conflicting run", + stages: [{ id: "other", title: "Other", instructions: "Do other work" }], + }), + /workspace lineage/, + ) + run = (await reloaded.approve(started.id, "implementer"))! + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await reloaded.get(started.id))! + } + assert.equal(run.status, "completed") + const [restored] = await reloaded.list("workspace-restored") + assert.equal(restored?.workspaceId, "workspace-restored") + } finally { + await manager.shutdown() + await reloaded?.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("fails a stage when its OpenCode request exceeds the operation timeout", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-timeout-")) + let session = 0 + let aborts = 0 + const client = { + session: { + create: async () => ({ data: { id: `session-${++session}` } }), + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => new Promise((_, reject) => { + const signal = options?.signal + if (!signal) return + signal.addEventListener("abort", () => reject(signal.reason), { once: true }) + }), + abort: async () => ++aborts === 1 ? { error: "abort failed" } : { data: true }, + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage-timeout", path: "C:/timeout-workspace", status: "ready" }), + } as unknown as WorkspaceManager + const eventBus = { publish: () => true } as unknown as EventBus + const logger = { warn() {}, error() {} } as unknown as Logger + const manager = new WorkflowManager({ + workspaceManager, + eventBus, + logger, + storageDir, + createClient: () => client, + promptTimeoutMs: 10, + }) + + try { + const started = await manager.start({ + workspaceId: "workspace", + objective: "Never finish", + stages: [{ id: "blocked", title: "Blocked", instructions: "Wait forever" }], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + assert.equal(run.status, "recovery_required") + assert.equal(run.steps[0]?.status, "failed") + assert.equal(aborts, 1) + await assert.rejects(manager.start({ + workspaceId: "workspace", + objective: "Must remain blocked", + stages: [{ id: "next", title: "Next", instructions: "Do not start" }], + }), /already running/) + assert.equal((await manager.cancel(started.id))?.status, "cancelled") + assert.equal(aborts, 2) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("keeps force-created instances of the same path isolated by lineage", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-lineage-")) + let session = 0 + const client = { + session: { + create: async () => ({ data: { id: `session-${++session}` } }), + prompt: async () => ({ data: { info: {}, parts: [{ type: "text", text: "Review me" }] } }), + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: (id: string) => ({ id, lineageId: id === "a" ? "lineage-a" : "lineage-b", path: "C:/same", status: "ready" }), + } as unknown as WorkspaceManager + const eventBus = { publish: () => true } as unknown as EventBus + const logger = { warn() {}, error() {} } as unknown as Logger + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir, createClient: () => client }) + + try { + const started = await manager.start({ + workspaceId: "a", + objective: "Lineage A", + stages: [{ id: "only", title: "Only", instructions: "Run", requiresApproval: true }], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + assert.equal(run.status, "waiting_for_review") + assert.deepEqual(await manager.list("b"), []) + assert.equal(await manager.get(started.id, "b"), undefined) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("does not fail a completed run when history pruning fails", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-prune-")) + let session = 0 + const client = { + session: { + create: async () => ({ data: { id: `session-${++session}` } }), + prompt: async () => ({ data: { info: {}, parts: [{ type: "text", text: "Complete" }] } }), + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }), + } as unknown as WorkspaceManager + let warnings = 0 + const logger = { warn: () => { warnings += 1 }, error() {} } as unknown as Logger + const manager = new WorkflowManager({ + workspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger, + storageDir, + createClient: () => client, + }) + ;(manager as any).pruneHistory = async () => { throw new Error("prune failed") } + + try { + const started = await manager.start({ + workspaceId: "workspace", + objective: "Finish despite prune failure", + stages: [{ id: "only", title: "Only", instructions: "Complete" }], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "completed") + assert.equal(warnings, 1) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("clears a rejected cached shutdown so shutdown can be retried", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-shutdown-retry-")) + const manager = new WorkflowManager({ + workspaceManager: { list: () => [] } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + await manager.list() + let attempts = 0 + ;(manager as any).performShutdown = async () => { + if (++attempts === 1) throw new Error("shutdown failed") + } + try { + await assert.rejects(manager.shutdown(), /shutdown failed/) + await assert.doesNotReject(manager.shutdown()) + assert.equal(attempts, 2) + } finally { + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workflows/manager.ts b/packages/server/src/workflows/manager.ts new file mode 100644 index 000000000..34c59c2d5 --- /dev/null +++ b/packages/server/src/workflows/manager.ts @@ -0,0 +1,1764 @@ +import { randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import path from "node:path" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { + WorkflowDefinitionV1, + WorkflowDefinitionRecord, + WorkflowDefinitionRunCreateRequest, + WorkflowNode, + WorkflowRun, + WorkflowRunStartRequest, + WorkflowRunStep, + WorkflowRunWorktreePolicy, + WorkflowRunWorktreeSelection, + WorkflowSavedDefinitionSnapshot, + WorkspaceDescriptor, +} from "../api-types" +import type { EventBus } from "../events/bus" +import type { Logger } from "../logger" +import { createInstanceClient } from "../workspaces/instance-client" +import { createManagedWorktree, isManagedWorktree, isValidWorktreeSlug, listWorktrees, resolveRepoRoot } from "../workspaces/git-worktrees" +import type { WorkspaceManager } from "../workspaces/manager" +import { ensureCodenomadGitExclude } from "../workspaces/worktree-map" +import { WorkflowDefinitionStore } from "./definition-store" +import { validateWorkflowDefinition, WORKFLOW_LIMITS } from "./definition-schema" +import { WorkflowInterpreter, WorkflowSuspendedError } from "./interpreter" +import { validateJsonSchemaValue } from "./json-schema" +import { definitionRunFields, holdsWorkflowReservation, markWorkflowRecoveryRequired, validatePersistedWorkflowRun } from "./run-state" + +const PROMPT_TIMEOUT_MS = 30 * 60 * 1000 +const ABORT_TIMEOUT_MS = 5_000 +const SHUTDOWN_TIMEOUT_MS = 10_000 +const MAX_OUTPUT_CHARS = 16_000 +const WORKFLOW_HISTORY_LIMIT = 100 +const CREATION_CLEANUP_ATTEMPTS = 3 + +export class WorkflowRunError extends Error { + constructor(message: string, readonly statusCode: number) { + super(message) + } +} + +interface WorkflowManagerOptions { + workspaceManager: WorkspaceManager + eventBus: EventBus + logger: Logger + storageDir: string + definitionsDir?: string + createClient?: (workspaceId: string) => OpencodeClient | null + promptTimeoutMs?: number +} + +interface ActiveRun { + run: WorkflowRun + client: OpencodeClient + activeSessionId?: string + activeSessionIds: Set + abortingSessions: Map> + cancelRequested: boolean + completion?: Promise + abortController: AbortController + releaseBlocked: boolean + pauseCommitted: boolean +} + +interface DefinitionGraphSnapshot { + root: WorkflowDefinitionV1 + saved: WorkflowSavedDefinitionSnapshot[] +} + +interface RunIndexEntry { + id: string + workspaceId: string + workspaceLineageId: string + workspacePath: string + sourceWorkspaceId?: string + sourceWorkspaceLineageId?: string + sourceWorkspacePath?: string + worktreeDirectory?: string + worktreeSlug?: string + status: WorkflowRun["status"] + createdAt: string + updatedAt: string + ambiguousSessions: boolean +} + +interface RunFileMetadata extends RunIndexEntry { + size: number + mtimeMs: number +} + +export interface WorkflowWorkspaceIdentity { + id?: string + lineageId?: string + path?: string +} + +export interface WorkflowWorktreeIdentity { + slug?: string + path?: string +} + +class WorkflowCancelledError extends Error {} + +export class WorkflowManager { + private readonly activeRuns = new Map() + private readonly activeWorkspaces = new Map() + private readonly reservedLineages = new Map() + private readonly reservedPaths = new Map() + private readonly persistQueues = new Map>() + private readonly transitionQueues = new Map>() + private readonly deferredCreationCleanups = new Map() + private readonly runIndex = new Map() + private readonly shutdownAbortController = new AbortController() + private readonly createClient: (workspaceId: string) => OpencodeClient | null + private readonly promptTimeoutMs: number + private readonly initialized: Promise + private readonly definitionStore: WorkflowDefinitionStore + private admissionQueue = Promise.resolve() + private creationCleanupQueue = Promise.resolve() + private shuttingDown = false + private shutdownPromise?: Promise + private admissionFailure?: string + + constructor(private readonly options: WorkflowManagerOptions) { + this.promptTimeoutMs = options.promptTimeoutMs ?? PROMPT_TIMEOUT_MS + this.createClient = options.createClient + ?? ((workspaceId) => createInstanceClient(options.workspaceManager, workspaceId, { timeoutMs: WORKFLOW_LIMITS.timeoutMs })) + this.definitionStore = new WorkflowDefinitionStore( + options.definitionsDir ?? path.join(options.storageDir, "definitions"), + ) + this.initialized = this.recoverInterruptedRuns() + } + + async start(input: WorkflowRunStartRequest): Promise { + await this.initialized + this.throwIfShuttingDown() + try { + return await this.withAdmission(() => this.startAdmitted(input)) + } finally { + await this.drainCreationCleanups() + } + } + + async startLatest(input: WorkflowDefinitionRunCreateRequest): Promise { + await this.initialized + this.throwIfShuttingDown() + try { + return await this.withAdmission(async () => { + const current = await this.definitionStore.get(input.definitionId) + if (!current) throw new WorkflowRunError("Workflow definition not found", 404) + if (input.definitionRevision !== undefined && current.revision !== input.definitionRevision) { + throw new WorkflowRunError("Workflow definition revision is stale", 409) + } + return this.startAdmitted({ ...input, definitionRevision: current.revision }) + }) + } finally { + await this.drainCreationCleanups() + } + } + + private async startAdmitted(input: WorkflowRunStartRequest): Promise { + this.throwIfShuttingDown() + this.throwIfAdmissionBlocked() + const legacy = "stages" in input + const definition = legacy ? undefined : await this.definitionStore.get(input.definitionId, input.definitionRevision) + if (!legacy && !definition) throw new WorkflowRunError("Workflow definition not found", 404) + const graph = definition ? await this.snapshotDefinitionGraph(definition) : undefined + this.throwIfShuttingDown() + const selected = legacy + ? this.currentWorkspace(input.workspaceId) + : await this.selectWorktree(input, input.worktree ?? { mode: "current" }) + const workspace = selected.workspace + const creationRequest = "creationRequest" in selected ? selected.creationRequest : undefined + let active: ActiveRun | undefined + let persisted = false + let retained = !creationRequest + try { + const client = this.requireReadyClient(workspace.id) + const now = new Date().toISOString() + const run: WorkflowRun = { + id: randomUUID(), + workspaceId: workspace.id, + workspaceLineageId: workspace.lineageId ?? workspace.id, + workspacePath: workspace.path, + ...(input.initiatorSessionId ? { initiatorSessionId: input.initiatorSessionId } : {}), + objective: input.objective ?? definition!.definition.name, + status: "running", + steps: legacy ? input.stages.map((stage) => ({ ...stage, status: "pending" })) : [], + revision: 0, + ...(definition && !("stages" in input) ? { + ...definitionRunFields(definition, input), + definitionSnapshot: graph!.root, + savedDefinitionSnapshots: graph!.saved, + worktreeSelection: selected.selection, + } : {}), + createdAt: now, + updatedAt: now, + } + active = { + run, client, activeSessionIds: new Set(), abortingSessions: new Map(), cancelRequested: false, + abortController: new AbortController(), releaseBlocked: false, pauseCommitted: false, + } + this.reserve(active) + await this.persist(run) + persisted = true + if (creationRequest) { + this.throwIfShuttingDown() + if (!this.options.workspaceManager.releaseCreationRequest(creationRequest.workspaceId, creationRequest.requestId)) { + throw new WorkflowRunError("Managed worktree workspace ownership could not be retained; the workflow was not started", 500) + } + retained = true + } + this.launch(active, (current) => legacy ? this.executePendingStages(current) : this.executeDefinition(current)) + return run + } catch (error) { + if (active && !retained) { + try { + if (persisted) await Promise.all([ + fs.rm(this.runPath(active.run.id), { force: true }), + fs.rm(this.runMetadataPath(active.run.id), { force: true }), + ]) + this.runIndex.delete(active.run.id) + this.release(active, true) + } catch (rollbackError) { + active.releaseBlocked = true + this.admissionFailure = `Workflow admission is blocked because startup rollback failed for run ${active.run.id}; remove or repair it and restart CodeNomad` + this.options.logger.error({ err: rollbackError, runId: active.run.id }, "Failed to roll back workflow startup") + } + } else if (active && !persisted) { + this.release(active, true) + } + throw error + } finally { + if (creationRequest && !retained) this.deferCreationCleanup(creationRequest.requestId) + } + } + + private currentWorkspace(workspaceId: string): { workspace: WorkspaceDescriptor; selection?: undefined } { + const workspace = this.options.workspaceManager.get(workspaceId) + if (!workspace) throw new WorkflowRunError("Workspace not found", 404) + if (workspace.status !== "ready") throw new WorkflowRunError("Workspace instance is not ready", 409) + return { workspace } + } + + private async selectWorktree( + input: WorkflowDefinitionRunCreateRequest, + policy: WorkflowRunWorktreePolicy, + ): Promise<{ + workspace: WorkspaceDescriptor + selection: WorkflowRunWorktreeSelection + creationRequest?: { workspaceId: string; requestId: string } + }> { + const source = this.currentWorkspace(input.workspaceId).workspace + if (policy.mode === "current") return { + workspace: source, + selection: { + policy, + sourceWorkspaceId: source.id, + sourceWorkspaceLineageId: source.lineageId ?? source.id, + sourceWorkspacePath: source.path, + workspaceId: source.id, + directory: source.path, + created: false, + }, + } + if (input.initiatorSessionId) { + throw new WorkflowRunError("initiatorSessionId is unsupported when a workflow selects a different worktree workspace", 400) + } + if (!isValidWorktreeSlug(policy.slug) || policy.slug === "root") { + throw new WorkflowRunError("Invalid workflow worktree slug", 400) + } + + let repository + try { + repository = await resolveRepoRoot(source.path, this.options.logger, { + signal: this.shutdownAbortController.signal, + }) + } catch (error) { + throw new WorkflowRunError(`Workflow worktree selection is unavailable: ${this.errorMessage(error)}`, 409) + } + const { repoRoot, isGitRepo } = repository + if (!isGitRepo) throw new WorkflowRunError("Workflow worktree policy requires a Git repository", 400) + let target: { slug: string; directory: string; branch?: string } + if (policy.mode === "existing") { + const match = (await listWorktrees({ + repoRoot, workspaceFolder: source.path, logger: this.options.logger, + signal: this.shutdownAbortController.signal, + })) + .find((worktree) => worktree.kind === "worktree" && worktree.slug === policy.slug) + if (!match) throw new WorkflowRunError(`Managed worktree ${policy.slug} was not found`, 404) + if (!await isManagedWorktree({ repoRoot, worktree: match })) { + throw new WorkflowRunError(`Worktree ${policy.slug} is not managed by CodeNomad`, 400) + } + target = match + } else { + this.throwIfShuttingDown() + await ensureCodenomadGitExclude(source.path, this.options.logger).catch(() => undefined) + this.throwIfShuttingDown() + try { + target = await createManagedWorktree({ + repoRoot, workspaceFolder: source.path, slug: policy.slug, logger: this.options.logger, + signal: this.shutdownAbortController.signal, + }) + } catch (error) { + throw new WorkflowRunError(`Failed to create managed worktree ${policy.slug}: ${this.errorMessage(error)}`, 409) + } + } + + const requestId = `workflow-${randomUUID()}` + let handedOff = false + try { + this.throwIfShuttingDown() + const createdWorkspace = await this.options.workspaceManager.create(target.directory, `Workflow: ${policy.slug}`, { + requestId, + binaryPath: source.binaryId, + }) + this.throwIfShuttingDown() + handedOff = true + return { + workspace: createdWorkspace.workspace, + creationRequest: { workspaceId: createdWorkspace.workspace.id, requestId }, + selection: { + policy, + sourceWorkspaceId: source.id, + sourceWorkspaceLineageId: source.lineageId ?? source.id, + sourceWorkspacePath: source.path, + workspaceId: createdWorkspace.workspace.id, + directory: createdWorkspace.workspace.path, + slug: target.slug, + ...(target.branch ? { branch: target.branch } : {}), + created: policy.mode === "new", + }, + } + } catch (error) { + if (error instanceof WorkflowRunError) throw error + throw new WorkflowRunError( + `Managed worktree ${policy.slug} was selected, but its OpenCode workspace could not be started: ${this.errorMessage(error)}. The worktree was retained for inspection.`, + 409, + ) + } finally { + if (!handedOff) this.deferCreationCleanup(requestId) + } + } + + private async snapshotDefinitionGraph(root: WorkflowDefinitionRecord): Promise { + const saved = new Map() + const resolved = new Set() + const snapshot = async (record: WorkflowDefinitionRecord, stack: string[], depth: number): Promise => { + if (depth > WORKFLOW_LIMITS.nestingDepth) { + throw new WorkflowRunError(`Saved workflow nesting exceeds maximum depth ${WORKFLOW_LIMITS.nestingDepth}`, 400) + } + if (stack.includes(record.id)) { + throw new WorkflowRunError(`Saved workflow cycle detected: ${[...stack, record.id].join(" -> ")}`, 400) + } + const key = `${record.id}@${record.revision}` + if (resolved.has(key)) return saved.get(key)!.definition + const definition = JSON.parse(JSON.stringify(record.definition)) as WorkflowDefinitionV1 + const nextStack = [...stack, record.id] + const inspect = async (node: WorkflowNode): Promise => { + if (node.type === "workflow") { + const child = await this.definitionStore.get(node.definitionId, node.definitionRevision) + if (!child) { + const revision = node.definitionRevision ? ` revision ${node.definitionRevision}` : "" + throw new WorkflowRunError(`Referenced workflow definition ${node.definitionId}${revision} was not found`, 404) + } + node.definitionRevision = child.revision + const childDefinition = await snapshot(child, nextStack, depth + 1) + const childKey = `${child.id}@${child.revision}` + if (!saved.has(childKey)) saved.set(childKey, { id: child.id, revision: child.revision, definition: childDefinition }) + if (saved.size > WORKFLOW_LIMITS.staticNodes) { + throw new WorkflowRunError(`Saved workflow graph exceeds ${WORKFLOW_LIMITS.staticNodes} definitions`, 400) + } + return + } + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + for (const child of children) await inspect(child) + } + await inspect(definition.root) + resolved.add(key) + return definition + } + const rootDefinition = await snapshot(root, [], 0) + saved.delete(`${root.id}@${root.revision}`) + const snapshots = Array.from(saved.values()) + const graphBytes = Buffer.byteLength(JSON.stringify([rootDefinition, ...snapshots.map((item) => item.definition)]), "utf8") + if (graphBytes > WORKFLOW_LIMITS.sourceBytes) { + throw new WorkflowRunError(`Saved workflow graph exceeds ${WORKFLOW_LIMITS.sourceBytes} bytes`, 400) + } + const byKey = new Map(snapshots.map((item) => [`${item.id}@${item.revision}`, item.definition])) + const limit = rootDefinition.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + const add = (left: number, right: number) => Math.min(limit + 1, left + right) + const multiply = (left: number, right: number) => Math.min(limit + 1, left * right) + const expansionMemo = new Map() + const definitionExpansion = (key: string, definition: WorkflowDefinitionV1): number => { + const cached = expansionMemo.get(key) + if (cached !== undefined) return cached + const result = expansion(definition.root) + expansionMemo.set(key, result) + return result + } + const expansion = (node: WorkflowNode): number => { + if (node.type === "workflow") { + const key = `${node.definitionId}@${node.definitionRevision}` + const child = byKey.get(key) + return child ? add(1, definitionExpansion(key, child)) : limit + 1 + } + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + let total = 1 + for (const child of children) { + total = add(total, expansion(child)) + if (total > limit) return total + } + if (node.type === "foreach") total = add(1, multiply(total - 1, node.maxItems)) + if (node.type === "repeat") total = add(1, multiply(total - 1, node.maxIterations)) + return total + } + const expanded = expansion(rootDefinition.root) + if (expanded > limit) throw new WorkflowRunError(`Saved workflow graph can expand above limit ${limit}`, 400) + return { root: rootDefinition, saved: snapshots } + } + + validateDefinition(source: string | unknown) { return validateWorkflowDefinition(source) } + async createDefinition(source: string | unknown) { + await this.initialized + return this.withAdmission(() => this.definitionStore.create(source)) + } + async updateDefinition(id: string, expectedRevision: number, source: string | unknown) { + await this.initialized + return this.withAdmission(() => this.definitionStore.update(id, expectedRevision, source)) + } + async deleteDefinition(id: string, expectedRevision: number) { + await this.initialized + return this.withAdmission(() => this.definitionStore.delete(id, expectedRevision)) + } + async getDefinition(id: string, revision?: number) { return this.definitionStore.get(id, revision) } + async listDefinitions(): Promise { return this.definitionStore.list() } + + async get(runId: string, workspaceId?: string): Promise { + await this.initialized + const active = this.activeRuns.get(runId) + if (!active) { + const run = await this.read(runId) + return workspaceId && run ? this.bindWorkspace(run.id, workspaceId) : run + } + if (["paused", "waiting_for_review", "waiting_for_input", "completed", "failed", "cancelled", "interrupted", "recovery_required"].includes(active.run.status)) { + await active.completion + if (active.releaseBlocked) return workspaceId ? this.bindWorkspace(active.run.id, workspaceId) : active.run + const run = await this.read(runId) ?? active.run + return workspaceId ? this.bindWorkspace(run.id, workspaceId) : run + } + return workspaceId ? this.bindWorkspace(active.run.id, workspaceId) : active.run + } + + async list(workspaceId?: string): Promise { + await this.initialized + const requested = workspaceId ? this.options.workspaceManager.get(workspaceId) : undefined + const candidates = Array.from(this.runIndex.values()) + .filter((entry) => !workspaceId || this.indexMatchesWorkspace(entry, workspaceId, requested)) + .sort((left, right) => right.createdAt.localeCompare(left.createdAt)) + const matched: WorkflowRun[] = [] + for (const entry of candidates) { + if (matched.length >= WORKFLOW_HISTORY_LIMIT) break + const run = await this.readForListing(`${entry.id}.json`) + if (!run) continue + const bound = workspaceId ? await this.bindWorkspace(run.id, workspaceId) : run + if (bound) matched.push(bound) + } + return matched + } + + /** Uncapped ownership predicate matching execution and source workspaces by ID, canonical lineage, or canonical path. */ + async isWorkspaceWorkflowOwned(identity: WorkflowWorkspaceIdentity): Promise { + await this.initialized + return this.withAdmission(() => this.isWorkspaceWorkflowOwnedCurrent(identity)) + } + + /** Holds the same serialized lease as workflow admission through the ownership decision and caller operation. */ + async withWorkspaceOwnershipLease( + identity: WorkflowWorkspaceIdentity, + operation: (owned: boolean) => Promise, + ): Promise { + await this.initialized + return this.withAdmission(async () => operation(await this.isWorkspaceWorkflowOwnedCurrent(identity))) + } + + /** Uncapped worktree predicate matching the source canonically and the selected worktree by slug or path. */ + async isWorktreeWorkflowOwned(source: WorkflowWorkspaceIdentity, worktree: WorkflowWorktreeIdentity): Promise { + await this.initialized + return this.withAdmission(() => this.isWorktreeWorkflowOwnedCurrent(source, worktree)) + } + + /** Holds workflow admission while a caller checks and deletes a canonical managed worktree. */ + async withWorktreeOwnershipLease( + source: WorkflowWorkspaceIdentity, + worktree: WorkflowWorktreeIdentity, + operation: (owned: boolean) => Promise, + ): Promise { + await this.initialized + return this.withAdmission(async () => operation(await this.isWorktreeWorkflowOwnedCurrent(source, worktree))) + } + + async approve(runId: string, expectedStepId: string): Promise { + await this.initialized + this.throwIfShuttingDown() + const candidate = this.activeRuns.get(runId)?.run ?? await this.read(runId) + if (candidate?.definitionSnapshot) { + if (candidate.pendingGate?.gate !== "approval") throw new WorkflowRunError("Workflow run is not waiting for approval", 409) + if (candidate.pendingGate.executionNodeId !== expectedStepId) throw new WorkflowRunError("Workflow approval is stale", 409) + return this.answer(runId, candidate.pendingGate.executionNodeId, true) + } + return this.withAdmission(() => this.withRunTransition(runId, async () => { + this.throwIfShuttingDown() + const current = this.activeRuns.get(runId) + if (current?.run.status === "waiting_for_review") await current.completion + else if (current) { + throw new WorkflowRunError("Workflow stage is not ready for review", 409) + } + const run = await this.read(runId) + if (!run) return undefined + if (run.pendingReviewStepId !== expectedStepId) throw new WorkflowRunError("Workflow approval is stale", 409) + const reviewed = run.steps.find((step) => step.id === run.pendingReviewStepId) + if (run.status !== "waiting_for_review" || reviewed?.status !== "completed") { + throw new WorkflowRunError("Workflow run is not waiting for review", 409) + } + + const restoredWorkspace = this.options.workspaceManager.list().find((workspace) => + workspace.lineageId === run.workspaceLineageId && workspace.status === "ready") + if (restoredWorkspace && restoredWorkspace.id !== run.workspaceId) { + await this.bindWorkspaceCurrent(run, restoredWorkspace.id) + } + const client = this.requireReadyClient(run.workspaceId, run.id) + const prior = this.cloneRun(run) + run.status = "running" + delete run.pendingReviewStepId + delete run.error + const active: ActiveRun = { + run, client, activeSessionIds: new Set(), abortingSessions: new Map(), cancelRequested: false, + abortController: new AbortController(), releaseBlocked: false, pauseCommitted: false, + } + this.reserve(active) + try { + await this.persist(run) + } catch (error) { + this.restoreRun(run, prior) + this.release(active) + this.reserveRun(run) + throw error + } + if (active.cancelRequested || this.shuttingDown) { + this.release(active) + return run + } + this.launch(active, (current) => this.executePendingStages(current)) + return run + })) + } + + async answer(runId: string, executionNodeId: string, answer: unknown): Promise { + await this.initialized + this.throwIfShuttingDown() + return this.withAdmission(() => this.withRunTransition(runId, async () => { + this.throwIfShuttingDown() + const current = this.activeRuns.get(runId) + if (current && (!current.run.pendingGate || !["waiting_for_review", "waiting_for_input"].includes(current.run.status))) { + if (current.run.executionNodes?.some((node) => node.id === executionNodeId && node.status === "completed")) { + throw new WorkflowRunError("Workflow gate answer is stale", 409) + } + throw new WorkflowRunError("Workflow run is not waiting for a gate answer", 409) + } + if (current?.completion) await current.completion + const run = await this.read(runId) + if (!run) return undefined + const gate = run.pendingGate + if (!run.definitionSnapshot || !gate || !["waiting_for_review", "waiting_for_input"].includes(run.status)) { + throw new WorkflowRunError("Workflow run is not waiting for a gate answer", 409) + } + if (gate.executionNodeId !== executionNodeId) throw new WorkflowRunError("Workflow gate answer is stale", 409) + if (gate.gate === "approval" && answer !== true) { + throw new WorkflowRunError("Approval gates require answer true", 400) + } + if (gate.gate === "input" && gate.inputSchema) { + const issues = validateJsonSchemaValue(answer, gate.inputSchema) + if (issues.length) throw new WorkflowRunError(`Gate answer is invalid: ${issues.join("; ")}`, 400) + } + const serializedAnswer = JSON.stringify(answer) + if (serializedAnswer === undefined || serializedAnswer.length > MAX_OUTPUT_CHARS) throw new WorkflowRunError("Gate answer is too large", 400) + const execution = run.executionNodes?.find((node) => node.id === gate.executionNodeId) + if (!execution || execution.status !== "waiting") throw new WorkflowRunError("Workflow gate state is invalid", 409) + const client = await this.prepareDefinitionWorkspace(run) + this.throwIfShuttingDown() + const prior = this.cloneRun(run) + await this.confirmPersistedSessionAborts(run, client, "Workflow gate answer") + execution.status = "completed" + execution.output = answer + execution.completedAt = new Date().toISOString() + delete run.pendingGate + return this.resumeDefinitionRun(run, client, prior) + })) + } + + async pause(runId: string): Promise { + await this.initialized + this.throwIfShuttingDown() + return this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + if (!run) return undefined + if (!run.definitionSnapshot) throw new WorkflowRunError("Legacy workflows cannot be paused", 409) + if (run.status === "paused" || run.status === "pausing") return run + if (run.status !== "running") throw new WorkflowRunError("Workflow run is not running", 409) + const priorStatus = run.status + const priorPauseRequested = run.pauseRequested + run.pauseRequested = true + run.status = "pausing" + try { + await this.persist(run) + const active = this.activeRuns.get(runId) + if (active?.run === run) active.pauseCommitted = true + } catch (error) { + if (run.status === "pausing") run.status = priorStatus + if (run.pauseRequested === true) { + if (priorPauseRequested === undefined) delete run.pauseRequested + else run.pauseRequested = priorPauseRequested + } + throw error + } + return run + }) + } + + async resume(runId: string, confirmRecovery = false): Promise { + await this.initialized + this.throwIfShuttingDown() + return this.withAdmission(() => this.withRunTransition(runId, async () => { + this.throwIfAdmissionBlocked() + const active = this.activeRuns.get(runId) + if (active && !["paused", "interrupted", "recovery_required"].includes(active.run.status)) { + throw new WorkflowRunError("Workflow run cannot be resumed", 409) + } + if (active?.completion) await active.completion + const run = await this.read(runId) + if (!run) return undefined + if (!run.definitionSnapshot) throw new WorkflowRunError("Legacy workflows cannot be resumed", 409) + if (run.status === "recovery_required" && !confirmRecovery) { + throw new WorkflowRunError("Recovery confirmation is required before repeating an ambiguous side effect", 409) + } + if (!["paused", "interrupted", "recovery_required"].includes(run.status)) { + throw new WorkflowRunError("Workflow run cannot be resumed", 409) + } + const client = await this.prepareDefinitionWorkspace(run) + this.throwIfShuttingDown() + const prior = this.cloneRun(run) + if (run.status === "recovery_required") { + const sessionIds = this.persistedAmbiguousSessionIds(run) + if (sessionIds.size === 0) { + const message = "Workflow recovery has no persisted session IDs and cannot positively confirm termination; the interrupted action will not be repeated" + await this.persistMutation(run, () => markWorkflowRecoveryRequired(run, message)) + throw new WorkflowRunError(message, 409) + } + await this.confirmPersistedSessionAborts(run, client, "Workflow recovery") + for (const node of run.executionNodes ?? []) if (node.status === "interrupted") { + node.status = "pending" + node.attempt = 0 + delete node.error + delete node.completedAt + } + } else { + await this.confirmPersistedSessionAborts(run, client, "Workflow resume") + } + if (active) { + active.activeSessionIds.clear() + active.activeSessionId = undefined + active.releaseBlocked = false + if (this.activeRuns.get(run.id) === active) this.activeRuns.delete(run.id) + } + return this.resumeDefinitionRun(run, client, prior) + })) + } + + async cancel(runId: string): Promise { + await this.initialized + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + return this.cancelRun(runId) + } + + /** Atomically verifies plugin workspace ownership and cancels without joining/rebinding the run via get(). */ + async cancelOwned(runId: string, workspaceId: string): Promise { + await this.initialized + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + return this.withAdmission(() => this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + if (!run) return undefined + const workspace = this.options.workspaceManager.get(workspaceId) + const executionMatch = run.workspaceId === workspaceId || Boolean(workspace?.status === "ready" + && this.matchesCanonicalWorkspace(workspace, run.workspaceLineageId, run.workspacePath)) + const selection = run.worktreeSelection + const sourceMatch = selection?.sourceWorkspaceId === workspaceId || Boolean(selection && workspace?.status === "ready" + && this.matchesCanonicalWorkspace(workspace, selection.sourceWorkspaceLineageId, selection.sourceWorkspacePath)) + if (!executionMatch && !sourceMatch) return undefined + if (executionMatch && run.workspaceId !== workspaceId) { + if (!await this.bindWorkspaceCurrent(run, workspaceId)) return undefined + } + return this.cancelRunCurrent(runId, run) + })) + } + + async shutdown(): Promise { + if (!this.shutdownPromise) { + this.shuttingDown = true + this.shutdownAbortController.abort(new WorkflowCancelledError()) + this.shutdownPromise = this.performShutdown() + } + const pending = this.shutdownPromise + try { + await this.withTimeout(pending, SHUTDOWN_TIMEOUT_MS, "Workflow shutdown timed out") + } catch (error) { + if (this.shutdownPromise === pending) this.shutdownPromise = undefined + throw error + } + } + + private async cancelRun(runId: string): Promise { + return this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + if (!run) return undefined + return this.cancelRunCurrent(runId, run) + }) + } + + private async cancelRunCurrent(runId: string, run: WorkflowRun): Promise { + const active = this.activeRuns.get(runId) + if (!["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "interrupted", "recovery_required"].includes(run.status)) return run + + let terminationConfirmed = true + if (active) { + this.requestCancellation(active) + await this.drainSessions(active) + await active.completion + terminationConfirmed = await this.drainSessions(active) + if (terminationConfirmed) this.clearPersistedSessions(run, this.persistedAmbiguousSessionIds(run)) + } + const sessionIds = this.persistedAmbiguousSessionIds(run) + if (sessionIds.size > 0) { + const client = active?.client ?? await this.prepareDefinitionWorkspace(run) + terminationConfirmed = terminationConfirmed && (await Promise.all(Array.from(sessionIds).map((sessionId) => + this.abortSessionRequest(client, run.id, sessionId)))).every(Boolean) + } else if (run.status === "recovery_required" && !active) { + terminationConfirmed = false + } + const message = sessionIds.size === 0 + ? "Workflow cancellation has no persisted session IDs and cannot positively confirm termination" + : "Workflow cancellation could not confirm every session abort" + try { + await this.persistMutation(run, () => { + if (terminationConfirmed) { + this.clearPersistedSessions(run, sessionIds) + this.markCancelled(run) + } else { + markWorkflowRecoveryRequired(run, message) + } + }) + if (terminationConfirmed) await fs.rm(this.recoveryMarkerPath(run.id), { force: true }).catch((error) => { + this.options.logger.warn({ err: error, runId: run.id }, "Failed to clear cancelled workflow recovery marker") + }) + } catch (error) { + if (active) active.releaseBlocked = true + this.reserveRun(run) + throw error + } + if (active) { + active.releaseBlocked = !terminationConfirmed + if (terminationConfirmed) this.release(active) + } + if (!holdsWorkflowReservation(run) && this.activeWorkspaces.get(run.workspaceId) === run.id) { + this.activeWorkspaces.delete(run.workspaceId) + } + if (!holdsWorkflowReservation(run) && this.reservedLineages.get(run.workspaceLineageId) === run.id) { + this.reservedLineages.delete(run.workspaceLineageId) + } + if (!holdsWorkflowReservation(run) && this.reservedPaths.get(this.pathKey(run.workspacePath)) === run.id) { + this.reservedPaths.delete(this.pathKey(run.workspacePath)) + } + return run + } + + private async performShutdown(): Promise { + await this.initialized + await this.admissionQueue.catch(() => undefined) + await this.drainCreationCleanups(true) + const active = Array.from(this.activeRuns.values()) + for (const run of active) this.requestCancellation(run) + await Promise.all(active.map(({ run }) => this.cancelRun(run.id))) + await Promise.all(active.map(({ completion }) => completion)) + await Promise.all(Array.from(this.transitionQueues.values())) + await Promise.all(Array.from(this.persistQueues.values())) + } + + private async resumeDefinitionRun(run: WorkflowRun, client: OpencodeClient, prior = this.cloneRun(run)): Promise { + this.throwIfShuttingDown() + run.status = "running" + run.pauseRequested = false + delete run.error + const active: ActiveRun = { + run, + client, + activeSessionIds: new Set(), + abortingSessions: new Map(), + cancelRequested: false, + abortController: new AbortController(), + releaseBlocked: false, + pauseCommitted: false, + } + this.reserve(active) + try { + await this.persist(run) + await fs.rm(this.recoveryMarkerPath(run.id), { force: true }).catch((error) => { + this.options.logger.warn({ err: error, runId: run.id }, "Failed to clear resolved workflow recovery marker") + }) + } catch (error) { + this.restoreRun(run, prior) + active.releaseBlocked = false + this.release(active) + this.reserveRun(run) + throw error + } + this.launch(active, (current) => this.executeDefinition(current)) + return run + } + + private async prepareDefinitionWorkspace(run: WorkflowRun): Promise { + this.throwIfShuttingDown() + const lineageWorkspace = this.options.workspaceManager.list().find((workspace) => + workspace.status === "ready" && this.matchesCanonicalWorkspace(workspace, run.workspaceLineageId, run.workspacePath)) + if (lineageWorkspace && lineageWorkspace.id !== run.workspaceId) { + this.throwIfShuttingDown() + await this.bindWorkspaceCurrent(run, lineageWorkspace.id) + } + this.throwIfShuttingDown() + return this.requireReadyClient(run.workspaceId, run.id) + } + + private async withTimeout(operation: Promise, timeoutMs: number, message: string): Promise { + let timeout: NodeJS.Timeout | undefined + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs) + }), + ]) + } finally { + if (timeout) clearTimeout(timeout) + } + } + + private requireReadyClient(workspaceId: string, runId?: string): OpencodeClient { + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + this.throwIfAdmissionBlocked() + const workspace = this.options.workspaceManager.get(workspaceId) + if (!workspace) throw new WorkflowRunError("Workspace not found", 404) + if (workspace.status !== "ready") throw new WorkflowRunError("Workspace instance is not ready", 409) + const activeRunId = this.activeWorkspaces.get(workspaceId) + if (activeRunId && activeRunId !== runId) { + throw new WorkflowRunError("A workflow is already running in this workspace", 409) + } + const lineageId = workspace.lineageId ?? workspace.id + const lineageRunId = this.reservedLineages.get(lineageId) + if (lineageRunId && lineageRunId !== runId) { + throw new WorkflowRunError("A workflow is already running for this workspace lineage", 409) + } + const pathRunId = this.reservedPaths.get(this.pathKey(workspace.path)) + if (pathRunId && pathRunId !== runId) { + throw new WorkflowRunError("A workflow is already running for this workspace path", 409) + } + const client = this.createClient(workspaceId) + if (!client) throw new WorkflowRunError("Workspace instance is not ready", 409) + return client + } + + private throwIfShuttingDown(): void { + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + } + + private throwIfAdmissionBlocked(): void { + if (this.admissionFailure) throw new WorkflowRunError(this.admissionFailure, 503) + } + + private requestCancellation(active: ActiveRun): void { + active.releaseBlocked = true + active.cancelRequested = true + active.abortController.abort(new WorkflowCancelledError("Workflow run cancelled")) + } + + private async drainSessions(active: ActiveRun): Promise { + const sessionIds = new Set(active.activeSessionIds) + if (active.activeSessionId) sessionIds.add(active.activeSessionId) + const results = await Promise.all(Array.from(sessionIds).map((sessionId) => this.abortActiveSession(active, sessionId))) + return results.every(Boolean) && active.activeSessionIds.size === 0 && active.activeSessionId === undefined + } + + private reserve(active: ActiveRun) { + this.activeRuns.set(active.run.id, active) + // ponytail: one run per workspace; add worktree-aware concurrency only when parallel workflows are needed. + this.reserveRun(active.run) + } + + private reserveRun(run: WorkflowRun) { + this.activeWorkspaces.set(run.workspaceId, run.id) + this.reservedLineages.set(run.workspaceLineageId, run.id) + this.reservedPaths.set(this.pathKey(run.workspacePath), run.id) + } + + private release(active: ActiveRun, force = false) { + if (this.activeRuns.get(active.run.id) !== active) return + if (!force && active.run.definitionSnapshot && this.persistedAmbiguousSessionIds(active.run).size > 0) active.releaseBlocked = true + if (active.releaseBlocked && !force) { + this.options.logger.error({ runId: active.run.id }, "Retaining workflow reservation after unconfirmed session abort") + return + } + this.activeRuns.delete(active.run.id) + if ((force || !holdsWorkflowReservation(active.run)) && this.activeWorkspaces.get(active.run.workspaceId) === active.run.id) { + this.activeWorkspaces.delete(active.run.workspaceId) + } + if ((force || !holdsWorkflowReservation(active.run)) && this.reservedLineages.get(active.run.workspaceLineageId) === active.run.id) { + this.reservedLineages.delete(active.run.workspaceLineageId) + } + const pathKey = this.pathKey(active.run.workspacePath) + if ((force || !holdsWorkflowReservation(active.run)) && this.reservedPaths.get(pathKey) === active.run.id) { + this.reservedPaths.delete(pathKey) + } + } + + private withAdmission(operation: () => Promise): Promise { + const admitted = this.admissionQueue.catch(() => undefined).then(operation) + this.admissionQueue = admitted.then(() => undefined, () => undefined) + return admitted + } + + private deferCreationCleanup(requestId: string): void { + if (!this.deferredCreationCleanups.has(requestId)) this.deferredCreationCleanups.set(requestId, 0) + } + + private async drainCreationCleanups(retryUntilExhausted = false): Promise { + const pending = this.creationCleanupQueue.catch(() => undefined).then(async () => { + do { + const cleanups = Array.from(this.deferredCreationCleanups) + if (cleanups.length === 0) return + for (const [requestId, attempts] of cleanups) { + try { + await this.options.workspaceManager.cancelCreationRequest(requestId) + this.deferredCreationCleanups.delete(requestId) + } catch (error) { + const nextAttempts = attempts + 1 + if (nextAttempts >= CREATION_CLEANUP_ATTEMPTS) this.deferredCreationCleanups.delete(requestId) + else this.deferredCreationCleanups.set(requestId, nextAttempts) + this.options.logger.warn( + { err: error, requestId, attempt: nextAttempts }, + nextAttempts >= CREATION_CLEANUP_ATTEMPTS + ? "Workflow workspace creation cleanup exhausted retries" + : "Workflow workspace creation cleanup will be retried", + ) + } + } + if (!retryUntilExhausted) return + } while (this.deferredCreationCleanups.size > 0) + }) + this.creationCleanupQueue = pending.then(() => undefined, () => undefined) + await pending + } + + private async withRunTransition(runId: string, transition: () => Promise): Promise { + const previous = this.transitionQueues.get(runId) ?? Promise.resolve() + const queued = previous.catch(() => undefined).then(transition) + const marker = queued.then(() => undefined, () => undefined) + this.transitionQueues.set(runId, marker) + try { + return await queued + } finally { + if (this.transitionQueues.get(runId) === marker) this.transitionQueues.delete(runId) + } + } + + private launch(active: ActiveRun, execute: (active: ActiveRun) => Promise) { + active.completion = execute(active) + .catch((error) => this.handleExecutionError(active, error)) + .catch(async (error) => { + const message = `Workflow recovery is required because terminal state could not be persisted: ${this.errorMessage(error)}` + const blockedMessage = `Workflow admission is blocked because recovery state could not be recorded for run ${active.run.id}; repair storage and restart CodeNomad` + markWorkflowRecoveryRequired(active.run, message) + active.releaseBlocked = true + this.reserveRun(active.run) + if (!this.admissionFailure) this.admissionFailure = blockedMessage + await this.writeRecoveryMarker(active.run, message).then(() => { + if (this.admissionFailure === blockedMessage) this.admissionFailure = undefined + }).catch((markerError) => { + this.options.logger.error({ err: markerError, runId: active.run.id }, "Failed to write workflow recovery marker") + }) + this.options.logger.error({ err: error, runId: active.run.id }, "Failed to persist workflow failure") + }) + .finally(() => this.release(active)) + } + + private async executePendingStages(active: ActiveRun): Promise { + const { run, client } = active + if (!run.rootSessionId) { + const root = await this.requireData(client.session.create({ + ...(run.initiatorSessionId ? { parentID: run.initiatorSessionId } : {}), + title: `Workflow: ${run.objective.slice(0, 80)}`, + metadata: this.sessionMetadata(run.id, "workflow"), + }, { signal: this.operationSignal(active) }), "create workflow session") + this.throwIfCancelled(active) + run.rootSessionId = root.id + await this.persist(run) + } + + while (true) { + const index = run.steps.findIndex((step) => step.status === "pending") + if (index < 0) break + const step = run.steps[index]! + const previous = index > 0 ? run.steps[index - 1]?.output : undefined + await this.runStep(active, step, this.buildStagePrompt(run, step, previous)) + this.throwIfCancelled(active) + if (step.requiresApproval) { + run.status = "waiting_for_review" + run.pendingReviewStepId = step.id + delete run.activeStepId + await this.persist(run) + return + } + } + + run.status = "completed" + delete run.activeStepId + delete run.pendingReviewStepId + await this.persist(run) + } + + private cloneRun(run: WorkflowRun): WorkflowRun { + return JSON.parse(JSON.stringify(run)) as WorkflowRun + } + + private restoreRun(run: WorkflowRun, snapshot: WorkflowRun): void { + const currentNodes = run.executionNodes + const restored = this.cloneRun(snapshot) + for (const key of Object.keys(run) as Array) delete run[key] + Object.assign(run, restored) + if (!currentNodes || !restored.executionNodes) return + const byId = new Map(currentNodes.map((node) => [node.id, node])) + const nodes = restored.executionNodes.map((snapshotNode) => { + const current = byId.get(snapshotNode.id) + if (!current) return snapshotNode + for (const key of Object.keys(current) as Array) delete current[key] + Object.assign(current, snapshotNode) + return current + }) + currentNodes.splice(0, currentNodes.length, ...nodes) + run.executionNodes = currentNodes + } + + private async persistMutation(run: WorkflowRun, mutate: () => void): Promise { + const prior = this.cloneRun(run) + mutate() + try { + await this.persist(run) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + } + + private async executeDefinition(active: ActiveRun): Promise { + const interpreter = new WorkflowInterpreter({ + run: active.run, + client: active.client, + persist: () => this.persist(active.run), + signal: (timeoutMs) => AbortSignal.any([active.abortController.signal, AbortSignal.timeout(timeoutMs)]), + sessionStarted: (sessionId) => { + active.activeSessionIds.add(sessionId) + return !active.cancelRequested + }, + sessionFinished: (sessionId) => active.activeSessionIds.delete(sessionId), + abortSession: async (sessionId) => { + const confirmed = await this.abortActiveSession(active, sessionId) + if (!confirmed) active.releaseBlocked = true + return confirmed + }, + isCancelled: () => active.cancelRequested, + isPauseCommitted: () => active.pauseCommitted, + }) + try { + await interpreter.execute() + } catch (error) { + if (error instanceof WorkflowSuspendedError) { + if (active.pauseCommitted && active.run.pauseRequested) { + const priorStatus = active.run.status + active.run.status = "paused" + try { + await this.persist(active.run) + } catch (persistError) { + if (active.run.status === "paused") active.run.status = priorStatus + throw persistError + } + } + return + } + throw error + } + this.throwIfCancelled(active) + active.run.status = "completed" + active.run.pauseRequested = false + delete active.run.pendingGate + await this.persist(active.run) + } + + private buildStagePrompt(run: WorkflowRun, step: WorkflowRunStep, previous: unknown): string { + return [ + `Workflow stage: ${step.title}`, + "", + `Objective:\n${run.objective}`, + "", + `Stage instructions:\n${step.instructions}`, + ...(previous === undefined ? [] : ["", `Previous stage handoff:\n${JSON.stringify(previous, null, 2)}`]), + ].join("\n") + } + + private async runStep( + active: ActiveRun, + step: WorkflowRunStep, + prompt: string, + ): Promise { + const { run, client } = active + this.throwIfCancelled(active) + step.status = "running" + step.startedAt = new Date().toISOString() + run.activeStepId = step.id + await this.persist(run) + this.throwIfCancelled(active) + + const session = await this.requireData(client.session.create({ + parentID: run.rootSessionId, + title: `${step.title}: ${run.objective.slice(0, 60)}`, + ...(step.agent ? { agent: step.agent } : {}), + metadata: this.sessionMetadata(run.id, step.id), + }, { signal: this.operationSignal(active) }), `create ${step.title} session`) + step.sessionId = session.id + active.activeSessionId = session.id + this.throwIfCancelled(active) + await this.persist(run) + this.throwIfCancelled(active) + + const response = await this.requireData(client.session.prompt({ + sessionID: session.id, + ...(step.agent ? { agent: step.agent } : {}), + ...(step.model ? { model: step.model } : {}), + parts: [{ type: "text", text: prompt }], + }, { signal: this.operationSignal(active) }), `run ${step.title} session`) + active.activeSessionId = undefined + this.throwIfCancelled(active) + if (response.info.error) throw new Error(this.errorMessage(response.info.error)) + + const output = response.info.structured ?? response.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n") + const bounded = this.boundOutput(output) + step.output = bounded.output + step.outputTruncated = bounded.truncated || undefined + step.status = "completed" + step.completedAt = new Date().toISOString() + active.activeSessionId = undefined + await this.persist(run) + return bounded.output + } + + private sessionMetadata(runId: string, role: string) { + return { codenomad: { version: 1, workflow: { runId, role } } } + } + + private operationSignal(active: ActiveRun): AbortSignal { + return AbortSignal.any([active.abortController.signal, AbortSignal.timeout(this.promptTimeoutMs)]) + } + + private boundOutput(output: unknown): { output: unknown; truncated: boolean } { + if (typeof output === "string") { + return output.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: output.slice(0, MAX_OUTPUT_CHARS), truncated: true } + } + const serialized = JSON.stringify(output) + return serialized.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: serialized.slice(0, MAX_OUTPUT_CHARS), truncated: true } + } + + private throwIfCancelled(active: ActiveRun) { + if (active.cancelRequested) throw new WorkflowCancelledError("Workflow run cancelled") + } + + private async abortSessionRequest(client: OpencodeClient, runId: string, sessionId: string): Promise { + try { + const response = await client.session.abort( + { sessionID: sessionId }, + { signal: AbortSignal.timeout(ABORT_TIMEOUT_MS) }, + ) + if (response.data === true && response.error === undefined) return true + this.options.logger.warn({ err: response.error, runId }, "Workflow session abort was not confirmed") + return false + } catch (error) { + this.options.logger.warn({ err: error, runId }, "Failed to abort workflow session") + return false + } + } + + private async abortActiveSession(active: ActiveRun, sessionId: string): Promise { + const existing = active.abortingSessions.get(sessionId) + if (existing) return existing + if (!active.activeSessionIds.has(sessionId) && active.activeSessionId !== sessionId) return true + const pending = this.abortSessionRequest(active.client, active.run.id, sessionId).then((confirmed) => { + if (confirmed) { + active.activeSessionIds.delete(sessionId) + if (active.activeSessionId === sessionId) active.activeSessionId = undefined + } + return confirmed + }).finally(() => active.abortingSessions.delete(sessionId)) + active.abortingSessions.set(sessionId, pending) + return pending + } + + private async handleExecutionError(active: ActiveRun, error: unknown): Promise { + const { run } = active + if (active.activeSessionId) { + const sessionId = active.activeSessionId + if (!await this.abortActiveSession(active, sessionId)) active.releaseBlocked = true + } + if (run.definitionSnapshot && !await this.drainSessions(active)) active.releaseBlocked = true + if (active.cancelRequested) return + if (active.releaseBlocked) { + const message = `Workflow session abort was not confirmed: ${this.errorMessage(error)}` + markWorkflowRecoveryRequired(run, message) + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "failed" + step.error = message + step.completedAt = new Date().toISOString() + } + await this.persist(run) + return + } + if (active.cancelRequested || error instanceof WorkflowCancelledError) { + this.markCancelled(run) + } else { + const message = this.errorMessage(error) + run.status = "failed" + run.error = message + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "failed" + step.error = message + step.completedAt = new Date().toISOString() + } + for (const node of run.executionNodes ?? []) { + if (node.status !== "running" && node.status !== "waiting") continue + node.status = "failed" + node.error = message + node.completedAt = new Date().toISOString() + } + delete run.pendingGate + delete run.activeStepId + this.options.logger.error({ err: error, runId: run.id }, "Workflow run failed") + } + await this.persist(run) + } + + private markCancelled(run: WorkflowRun) { + run.status = "cancelled" + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "cancelled" + step.completedAt = new Date().toISOString() + } + for (const node of run.executionNodes ?? []) { + if (node.status !== "running" && node.status !== "waiting") continue + node.status = "cancelled" + node.completedAt = new Date().toISOString() + } + run.pauseRequested = false + delete run.pendingGate + delete run.activeStepId + delete run.pendingReviewStepId + } + + private async requireData(request: Promise<{ data?: T; error?: unknown }>, action: string): Promise { + const response = await request + if (response.data !== undefined) return response.data + throw new Error(`${action} failed: ${this.errorMessage(response.error)}`) + } + + private errorMessage(error: unknown): string { + if (error instanceof Error) return error.message + if (typeof error === "string") return error + try { + return JSON.stringify(error) || "Unknown error" + } catch { + return "Unknown error" + } + } + + private runPath(runId: string) { + return path.join(this.options.storageDir, `${runId}.json`) + } + + private runMetadataPath(runId: string) { + return path.join(this.options.storageDir, `${runId}.meta`) + } + + private recoveryMarkerPath(runId: string) { + return path.join(this.options.storageDir, `${runId}.recovery`) + } + + private async writeRecoveryMarker(run: WorkflowRun, message: string): Promise { + await fs.mkdir(this.options.storageDir, { recursive: true }) + const destination = this.recoveryMarkerPath(run.id) + const temporary = `${destination}.${randomUUID()}.tmp` + await fs.writeFile(temporary, JSON.stringify({ runId: run.id, message, createdAt: new Date().toISOString() }), "utf8") + await fs.rename(temporary, destination) + } + + private async hasRecoveryMarker(runId: string): Promise { + try { + await fs.access(this.recoveryMarkerPath(runId)) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false + throw error + } + } + + private async bindWorkspace(runId: string, workspaceId: string): Promise { + return this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + return run ? this.bindWorkspaceCurrent(run, workspaceId) : undefined + }) + } + + private async bindWorkspaceCurrent(run: WorkflowRun, workspaceId: string): Promise { + const requested = this.options.workspaceManager.get(workspaceId) + const selection = run.worktreeSelection + if (run.workspaceId === workspaceId) { + if (selection?.policy.mode === "current" && selection.sourceWorkspaceId !== workspaceId) { + const prior = this.cloneRun(run) + selection.sourceWorkspaceId = workspaceId + try { + await this.persist(run, false) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + } + return run + } + if (selection && selection.policy.mode !== "current" && requested + && requested.status === "ready" + && (selection.sourceWorkspaceId === workspaceId || ( + requested.lineageId === selection.sourceWorkspaceLineageId + && this.samePath(requested.path, selection.sourceWorkspacePath) + ))) { + if (selection.sourceWorkspaceId !== workspaceId) { + const prior = this.cloneRun(run) + selection.sourceWorkspaceId = workspaceId + try { + await this.persist(run, false) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + } + return run + } + if (["running", "pausing"].includes(run.status)) return undefined + const workspace = requested + if (!workspace || !workspace.lineageId || run.workspaceLineageId !== workspace.lineageId) return undefined + if (!this.samePath(workspace.path, run.workspacePath)) return undefined + const prior = this.cloneRun(run) + const previousId = run.workspaceId + run.workspaceId = workspaceId + if (selection) { + selection.workspaceId = workspaceId + selection.directory = workspace.path + if (selection.policy.mode === "current") selection.sourceWorkspaceId = workspaceId + } + run.workspacePath = workspace.path + try { + await this.persist(run, false) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + if (this.activeWorkspaces.get(previousId) === run.id) this.activeWorkspaces.delete(previousId) + if (holdsWorkflowReservation(run)) this.activeWorkspaces.set(workspaceId, run.id) + return run + } + + private samePath(left: string, right: string): boolean { + const normalizedLeft = path.resolve(left) + const normalizedRight = path.resolve(right) + return process.platform === "win32" + ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase() + : normalizedLeft === normalizedRight + } + + private pathKey(value: string): string { + const resolved = path.resolve(value) + return process.platform === "win32" ? resolved.toLowerCase() : resolved + } + + private matchesCanonicalWorkspace(workspace: WorkspaceDescriptor, lineageId: string, workspacePath: string): boolean { + return workspace.lineageId === lineageId && this.samePath(workspace.path, workspacePath) + } + + private matchesIdentity(identity: WorkflowWorkspaceIdentity, id: string, lineageId: string, workspacePath: string): boolean { + return Boolean( + (identity.id && identity.id === id) + || (identity.lineageId && identity.lineageId === lineageId) + || (identity.path && this.samePath(identity.path, workspacePath)), + ) + } + + private async isWorkspaceWorkflowOwnedCurrent(identity: WorkflowWorkspaceIdentity): Promise { + if (this.admissionFailure) return true + if (identity.lineageId && this.reservedLineages.has(identity.lineageId)) return true + if (identity.path && this.reservedPaths.has(this.pathKey(identity.path))) return true + return Array.from(this.runIndex.values()).some((run) => { + if (!this.indexHoldsReservation(run)) return false + if (this.matchesIdentity(identity, run.workspaceId, run.workspaceLineageId, run.workspacePath)) return true + return Boolean(run.sourceWorkspaceId && run.sourceWorkspaceLineageId && run.sourceWorkspacePath && this.matchesIdentity( + identity, + run.sourceWorkspaceId, + run.sourceWorkspaceLineageId, + run.sourceWorkspacePath, + )) + }) + } + + private async isWorktreeWorkflowOwnedCurrent( + source: WorkflowWorkspaceIdentity, + worktree: WorkflowWorktreeIdentity, + ): Promise { + if (this.admissionFailure) return true + return Array.from(this.runIndex.values()).some((run) => { + if (!this.indexHoldsReservation(run) || !run.worktreeDirectory) return false + if (worktree.path && this.samePath(run.worktreeDirectory, worktree.path)) return true + if (!run.sourceWorkspaceId || !run.sourceWorkspaceLineageId || !run.sourceWorkspacePath || !this.matchesIdentity( + source, + run.sourceWorkspaceId, + run.sourceWorkspaceLineageId, + run.sourceWorkspacePath, + )) return false + return Boolean(worktree.slug && run.worktreeSlug === worktree.slug) + }) + } + + private persistedAmbiguousSessionIds(run: WorkflowRun): Set { + if (run.definitionSnapshot) return new Set((run.executionNodes ?? []) + .filter((node) => !["completed", "skipped", "failed", "cancelled"].includes(node.status)) + .flatMap((node) => node.sessionIds ?? [])) + return new Set(run.steps + .filter((step) => step.sessionId && (step.status === "running" || step.status === "failed")) + .map((step) => step.sessionId!)) + } + + private clearPersistedSessions(run: WorkflowRun, sessionIds: Set): void { + for (const node of run.executionNodes ?? []) { + if (!node.sessionIds) continue + node.sessionIds = node.sessionIds.filter((sessionId) => !sessionIds.has(sessionId)) + if (node.sessionIds.length === 0) delete node.sessionIds + } + for (const step of run.steps) if (step.sessionId && sessionIds.has(step.sessionId)) delete step.sessionId + } + + private async confirmPersistedSessionAborts(run: WorkflowRun, client: OpencodeClient, action: string): Promise { + const sessionIds = this.persistedAmbiguousSessionIds(run) + if (sessionIds.size === 0) return + const confirmed = await Promise.all(Array.from(sessionIds).map((sessionId) => + this.abortSessionRequest(client, run.id, sessionId))) + if (confirmed.some((result) => !result)) { + const message = `${action} could not confirm every persisted session abort` + await this.persistMutation(run, () => markWorkflowRecoveryRequired(run, message)) + throw new WorkflowRunError(message, 409) + } + this.clearPersistedSessions(run, sessionIds) + } + + private async read(runId: string): Promise { + try { + const run = JSON.parse(await fs.readFile(this.runPath(runId), "utf8")) as WorkflowRun + validatePersistedWorkflowRun(run, runId) + this.indexRun(run) + return run + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + throw error + } + } + + private async readForListing(entry: string): Promise { + try { + return await this.read(entry.slice(0, -5)) + } catch (error) { + this.options.logger.warn({ err: error, file: entry }, "Skipping corrupt workflow run") + return undefined + } + } + + private runIndexEntry(run: WorkflowRun): RunIndexEntry { + const selection = run.worktreeSelection + return { + id: run.id, + workspaceId: run.workspaceId, + workspaceLineageId: run.workspaceLineageId, + workspacePath: run.workspacePath, + ...(selection ? { + sourceWorkspaceId: selection.sourceWorkspaceId, + sourceWorkspaceLineageId: selection.sourceWorkspaceLineageId, + sourceWorkspacePath: selection.sourceWorkspacePath, + worktreeDirectory: selection.directory, + ...(selection.slug ? { worktreeSlug: selection.slug } : {}), + } : {}), + status: run.status, + createdAt: run.createdAt, + updatedAt: run.updatedAt, + ambiguousSessions: Boolean(run.definitionSnapshot && this.persistedAmbiguousSessionIds(run).size > 0), + } + } + + private indexRun(run: WorkflowRun): void { + this.runIndex.set(run.id, this.runIndexEntry(run)) + } + + private async readRunMetadata(entry: string): Promise { + const runId = entry.slice(0, -5) + try { + const [stored, stat] = await Promise.all([ + fs.readFile(this.runMetadataPath(runId), "utf8"), + fs.stat(path.join(this.options.storageDir, entry)), + ]) + const metadata = JSON.parse(stored) as RunFileMetadata + if (metadata.id !== runId || metadata.size !== stat.size || metadata.mtimeMs !== stat.mtimeMs + || typeof metadata.workspaceId !== "string" || typeof metadata.workspaceLineageId !== "string" + || typeof metadata.workspacePath !== "string" || typeof metadata.createdAt !== "string" + || typeof metadata.updatedAt !== "string" || typeof metadata.ambiguousSessions !== "boolean" + || !["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "completed", "failed", "cancelled", "interrupted", "recovery_required"] + .includes(metadata.status)) return undefined + return metadata + } catch { + return undefined + } + } + + private async writeRunMetadata(run: WorkflowRun): Promise { + const stat = await fs.stat(this.runPath(run.id)) + const metadata: RunFileMetadata = { ...this.runIndexEntry(run), size: stat.size, mtimeMs: stat.mtimeMs } + const destination = this.runMetadataPath(run.id) + const temporary = `${destination}.${randomUUID()}.tmp` + await fs.writeFile(temporary, JSON.stringify(metadata), "utf8") + await fs.rename(temporary, destination) + } + + private indexMatchesWorkspace(entry: RunIndexEntry, workspaceId: string, workspace?: WorkspaceDescriptor): boolean { + if (entry.workspaceId === workspaceId || entry.sourceWorkspaceId === workspaceId) return true + if (!workspace?.lineageId) return false + return this.matchesCanonicalWorkspace(workspace, entry.workspaceLineageId, entry.workspacePath) + || Boolean(entry.sourceWorkspaceLineageId && entry.sourceWorkspacePath + && this.matchesCanonicalWorkspace(workspace, entry.sourceWorkspaceLineageId, entry.sourceWorkspacePath)) + } + + private indexHoldsReservation(entry: RunIndexEntry): boolean { + return ["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "interrupted", "recovery_required"] + .includes(entry.status) + } + + private async recoverInterruptedRuns(): Promise { + let entries: string[] + try { + entries = await fs.readdir(this.options.storageDir) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + + for (const entry of entries.filter((candidate) => candidate.endsWith(".json"))) { + try { + const runId = entry.slice(0, -5) + const recoveryMarked = await this.hasRecoveryMarker(runId) + const metadata = await this.readRunMetadata(entry) + if (metadata && !recoveryMarked && !this.indexHoldsReservation(metadata) && !metadata.ambiguousSessions) { + this.runIndex.set(metadata.id, metadata) + continue + } + const run = await this.read(runId) + if (!run) continue + await this.writeRunMetadata(run).catch((error) => { + this.options.logger.warn({ err: error, runId: run.id }, "Failed to update workflow run metadata") + }) + if (recoveryMarked) { + const message = "Workflow terminal state was not durably persisted; manual recovery is required and prior gate state cannot be reused" + markWorkflowRecoveryRequired(run, message) + delete run.activeStepId + this.reserveRun(run) + try { + await this.persist(run) + await fs.rm(this.recoveryMarkerPath(run.id), { force: true }) + } catch (error) { + this.admissionFailure = `Workflow admission is blocked because marked recovery state could not be persisted for run ${run.id}; repair storage and restart CodeNomad` + throw error + } + continue + } + if (!holdsWorkflowReservation(run)) continue + const sessionBearing = this.persistedAmbiguousSessionIds(run).size > 0 + const interruptedAction = Boolean(run.definitionSnapshot && run.executionNodes?.some((node) => + node.status === "running" && (node.type === "agent" || node.type === "shell"))) + const legacyAction = !run.definitionSnapshot && run.steps.some((step) => step.status === "running") + const wasExecuting = run.status === "running" || run.status === "pausing" + const ambiguous = sessionBearing || interruptedAction || legacyAction + if (!wasExecuting && !ambiguous) { + this.reserveRun(run) + continue + } + run.error = "CodeNomad restarted before this workflow completed" + if (ambiguous) markWorkflowRecoveryRequired(run, run.error) + else run.status = "interrupted" + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "failed" + step.error = run.error + step.completedAt = new Date().toISOString() + } + for (const node of run.executionNodes ?? []) { + if ((node.sessionIds?.length && !["completed", "skipped", "failed", "cancelled"].includes(node.status)) + || (node.status === "running" && (node.type === "agent" || node.type === "shell"))) { + node.status = "interrupted" + node.error = run.error + node.completedAt = new Date().toISOString() + } else if (node.status === "running") { + node.status = "waiting" + } + } + if (run.status === "recovery_required" && this.persistedAmbiguousSessionIds(run).size === 0) { + run.error = "CodeNomad restarted during an action, but no session ID was persisted; termination cannot be positively confirmed and the action will not be repeated" + for (const node of run.executionNodes ?? []) if (node.status === "interrupted") node.error = run.error + if (step) step.error = run.error + } + this.reserveRun(run) + delete run.activeStepId + await this.persist(run) + } catch (error) { + await this.quarantineMalformedActiveRun(entry) + this.options.logger.error({ err: error, file: entry }, "Failed to recover workflow run") + } + } + await this.pruneHistoryEntries().catch((error) => { + this.options.logger.warn({ err: error }, "Failed to prune global workflow history") + }) + } + + private async quarantineMalformedActiveRun(entry: string): Promise { + let value: unknown + try { + value = JSON.parse(await fs.readFile(path.join(this.options.storageDir, entry), "utf8")) + } catch { + this.admissionFailure = `Workflow admission is blocked by malformed active run ${entry}; remove or repair it and restart CodeNomad` + return + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + this.admissionFailure = `Workflow admission is blocked by malformed active run ${entry}; remove or repair it and restart CodeNomad` + return + } + const candidate = value as Record + if (["completed", "failed", "cancelled"].includes(candidate.status as string)) return + const quarantineId = typeof candidate.id === "string" && candidate.id ? candidate.id : `malformed:${entry}` + let identifiable = false + if (typeof candidate.workspaceLineageId === "string" && candidate.workspaceLineageId) { + this.reservedLineages.set(candidate.workspaceLineageId, quarantineId) + identifiable = true + } + if (typeof candidate.workspacePath === "string" && candidate.workspacePath) { + this.reservedPaths.set(this.pathKey(candidate.workspacePath), quarantineId) + identifiable = true + } + if (typeof candidate.workspaceId === "string" && candidate.workspaceId) { + this.activeWorkspaces.set(candidate.workspaceId, quarantineId) + } + if (!identifiable) { + this.admissionFailure = `Workflow admission is blocked by malformed active run ${entry} without a usable lineage or path; remove or repair it and restart CodeNomad` + } + } + + private async persist(run: WorkflowRun, touch = true): Promise { + if (touch) { + run.updatedAt = new Date().toISOString() + run.revision = (run.revision ?? 0) + 1 + } + validatePersistedWorkflowRun(run, run.id) + const snapshot = JSON.parse(JSON.stringify(run)) as WorkflowRun + const previous = this.persistQueues.get(run.id) ?? Promise.resolve() + const queued = previous.catch(() => undefined).then(async () => { + await fs.mkdir(this.options.storageDir, { recursive: true }) + const destination = this.runPath(run.id) + const temporary = `${destination}.${randomUUID()}.tmp` + await fs.writeFile(temporary, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8") + await fs.rename(temporary, destination) + this.indexRun(snapshot) + await this.writeRunMetadata(snapshot).catch((error) => { + this.options.logger.warn({ err: error, runId: snapshot.id }, "Failed to update workflow run metadata") + }) + const workspaceIds = new Set([snapshot.workspaceId, snapshot.worktreeSelection?.sourceWorkspaceId].filter(Boolean) as string[]) + for (const instanceId of workspaceIds) this.options.eventBus.publish({ + type: "instance.event", + instanceId, + event: { type: "workflow.run.updated", properties: { run: snapshot } }, + }) + if (["completed", "failed", "cancelled"].includes(snapshot.status)) { + await this.pruneHistory(snapshot.worktreeSelection?.sourceWorkspaceLineageId ?? snapshot.workspaceLineageId).catch((error) => { + this.options.logger.warn({ err: error, runId: snapshot.id }, "Failed to prune workflow history") + }) + } + }) + this.persistQueues.set(run.id, queued) + try { + await queued + } finally { + if (this.persistQueues.get(run.id) === queued) this.persistQueues.delete(run.id) + } + } + + private async pruneHistory(workspaceLineageId: string): Promise { + return this.pruneHistoryEntries(workspaceLineageId) + } + + private async pruneHistoryEntries(workspaceLineageId?: string): Promise { + const terminal = Array.from(this.runIndex.values()) + .filter((run) => ["completed", "failed", "cancelled"].includes(run.status)) + .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)) + const lineage = workspaceLineageId + ? terminal.filter((run) => (run.sourceWorkspaceLineageId ?? run.workspaceLineageId) === workspaceLineageId) + : [] + const expired = new Set([ + ...terminal.slice(WORKFLOW_HISTORY_LIMIT).map((run) => run.id), + ...lineage.slice(WORKFLOW_HISTORY_LIMIT).map((run) => run.id), + ]) + await Promise.all(Array.from(expired).map(async (runId) => { + await Promise.all([fs.rm(this.runPath(runId), { force: true }), fs.rm(this.runMetadataPath(runId), { force: true })]) + this.runIndex.delete(runId) + })) + } +} diff --git a/packages/server/src/workflows/run-state.ts b/packages/server/src/workflows/run-state.ts new file mode 100644 index 000000000..074464fde --- /dev/null +++ b/packages/server/src/workflows/run-state.ts @@ -0,0 +1,171 @@ +import type { + WorkflowDefinitionV1, + WorkflowDefinitionRecord, + WorkflowDefinitionRunCreateRequest, + WorkflowNode, + WorkflowRun, + WorkflowUsage, +} from "../api-types" +import { parseWorkflowDefinition, WORKFLOW_LIMITS } from "./definition-schema" + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T + +export const emptyWorkflowUsage = (): WorkflowUsage => ({ + cost: 0, + tokens: 0, + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}) + +export const definitionRunFields = ( + record: WorkflowDefinitionRecord, + input: WorkflowDefinitionRunCreateRequest, +): Pick => ({ + definitionId: record.id, + definitionRevision: record.revision, + definitionSnapshot: clone(record.definition), + inputs: clone(input.inputs ?? {}), + executionNodes: [], + usage: emptyWorkflowUsage(), +}) + +export const holdsWorkflowReservation = (run: WorkflowRun) => + ["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "interrupted", "recovery_required"].includes(run.status) + +export function markWorkflowRecoveryRequired(run: WorkflowRun, message: string) { + run.status = "recovery_required" + run.error = message + for (const node of run.executionNodes ?? []) { + if (node.status !== "running" && node.status !== "waiting" + && !(node.sessionIds?.length && !["completed", "skipped", "failed", "cancelled"].includes(node.status))) continue + node.status = "interrupted" + node.error = message + node.completedAt = new Date().toISOString() + } + run.pauseRequested = false + delete run.pendingGate +} + +const RUN_STATUSES = new Set([ + "running", "pausing", "paused", "waiting_for_review", "waiting_for_input", + "completed", "failed", "cancelled", "interrupted", "recovery_required", +]) +const EXECUTION_STATUSES = new Set(["pending", "running", "waiting", "completed", "skipped", "failed", "cancelled", "interrupted"]) +const EXECUTION_TYPES = new Set(["sequence", "parallel", "foreach", "repeat", "agent", "shell", "gate", "condition", "workflow"]) +const isNonEmptyString = (value: unknown): value is string => typeof value === "string" && value.length > 0 +const isTimestamp = (value: unknown): value is string => isNonEmptyString(value) && Number.isFinite(Date.parse(value)) +const validUsage = (usage: WorkflowUsage) => Object.values(usage).every((value) => + typeof value === "number" && Number.isFinite(value) && value >= 0) + +export function validatePersistedWorkflowRun(value: unknown, runId: string): asserts value is WorkflowRun { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`Invalid stored workflow run ${runId}`) + const run = value as WorkflowRun + if (run.id !== runId + || !isNonEmptyString(run.workspaceId) + || !isNonEmptyString(run.workspaceLineageId) + || !isNonEmptyString(run.workspacePath) + || !isNonEmptyString(run.objective) + || !RUN_STATUSES.has(run.status) + || !isTimestamp(run.createdAt) + || !isTimestamp(run.updatedAt) + || !Array.isArray(run.steps)) { + throw new Error(`Invalid stored workflow run ${runId}`) + } + if (run.usage && !validUsage(run.usage)) throw new Error(`Invalid workflow usage for workflow run ${runId}`) + if (!run.definitionSnapshot) return + const { definition } = parseWorkflowDefinition(run.definitionSnapshot) + if (run.definitionId !== definition.id || !Number.isInteger(run.definitionRevision) || run.definitionRevision! < 1) { + throw new Error(`Invalid definition snapshot for workflow run ${runId}`) + } + const expandedLimit = definition.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + if (!Array.isArray(run.executionNodes) || run.executionNodes.length > expandedLimit) { + throw new Error(`Invalid execution nodes for workflow run ${runId}`) + } + if (new Set(run.executionNodes.map((node) => node.id)).size !== run.executionNodes.length + || new Set(run.executionNodes.map((node) => node.instanceKey)).size !== run.executionNodes.length) { + throw new Error(`Duplicate execution nodes for workflow run ${runId}`) + } + for (const node of run.executionNodes) { + if (!isNonEmptyString(node.id) || !isNonEmptyString(node.instanceKey) || !isNonEmptyString(node.definitionNodeId) + || (node.definitionInvocationKey !== undefined && !isNonEmptyString(node.definitionInvocationKey)) + || !EXECUTION_TYPES.has(node.type) || !EXECUTION_STATUSES.has(node.status) || !Number.isInteger(node.attempt) || node.attempt < 0 + || (node.sessionIds !== undefined && (!Array.isArray(node.sessionIds) || node.sessionIds.some((id) => !isNonEmptyString(id)))) + || (node.usage && !validUsage(node.usage))) { + throw new Error(`Invalid execution node for workflow run ${runId}`) + } + } + const snapshots = run.savedDefinitionSnapshots ?? [] + if (snapshots.length > WORKFLOW_LIMITS.staticNodes) throw new Error(`Too many saved definition snapshots for workflow run ${runId}`) + const byKey = new Map() + for (const snapshot of snapshots) { + const parsed = parseWorkflowDefinition(snapshot.definition).definition + if (snapshot.id !== parsed.id || !Number.isInteger(snapshot.revision) || snapshot.revision < 1) { + throw new Error(`Invalid saved definition snapshot for workflow run ${runId}`) + } + const key = `${snapshot.id}@${snapshot.revision}` + if (byKey.has(key)) throw new Error(`Duplicate saved definition snapshot for workflow run ${runId}`) + byKey.set(key, parsed) + } + const inspect = (node: typeof definition.root, stack: string[], depth: number): void => { + if (depth > WORKFLOW_LIMITS.nestingDepth) throw new Error(`Saved workflow nesting is too deep for workflow run ${runId}`) + if (node.type === "workflow") { + const key = `${node.definitionId}@${node.definitionRevision}` + const snapshot = byKey.get(key) + if (!snapshot || !node.definitionRevision) throw new Error(`Missing saved definition snapshot for workflow run ${runId}`) + if (stack.includes(snapshot.id)) throw new Error(`Saved workflow cycle in workflow run ${runId}`) + inspect(snapshot.root, [...stack, snapshot.id], depth + 1) + return + } + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + for (const child of children) inspect(child, stack, depth) + } + inspect(definition.root, [definition.id], 0) + const typesByDefinition = new Map>() + const collectTypes = (key: string, root: WorkflowNode) => { + const types = new Map() + const visit = (node: WorkflowNode): void => { + types.set(node.id, node.type) + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + for (const child of children) visit(child) + } + visit(root) + typesByDefinition.set(key, types) + } + const rootKey = `${run.definitionId}@${run.definitionRevision}` + collectTypes(rootKey, definition.root) + for (const [key, snapshot] of byKey) collectTypes(key, snapshot.root) + for (const node of run.executionNodes) { + const segments = node.instanceKey.split("/") + const savedInvocation = segments.map((segment, index) => ({ segment, index })) + .filter(({ segment }) => byKey.has(segment)).at(-1) + const definitionKey = savedInvocation?.segment ?? rootKey + const invocationKey = savedInvocation ? segments.slice(0, savedInvocation.index + 1).join("/") : rootKey + if (node.definitionInvocationKey !== undefined && node.definitionInvocationKey !== invocationKey) { + throw new Error(`Invalid definition invocation scope for workflow run ${runId}`) + } + if (typesByDefinition.get(definitionKey)?.get(node.definitionNodeId) !== node.type) { + throw new Error(`Execution node does not match the pinned graph for workflow run ${runId}`) + } + } + if (run.worktreeSelection) { + const selection = run.worktreeSelection + if (selection.workspaceId !== run.workspaceId || selection.directory !== run.workspacePath + || !selection.sourceWorkspaceId || !selection.sourceWorkspaceLineageId || !selection.sourceWorkspacePath) { + throw new Error(`Invalid worktree selection for workflow run ${runId}`) + } + if (selection.policy.mode !== "current" && selection.policy.slug !== selection.slug) { + throw new Error(`Invalid worktree selection for workflow run ${runId}`) + } + } +} diff --git a/packages/server/src/workflows/runtime.test.ts b/packages/server/src/workflows/runtime.test.ts new file mode 100644 index 000000000..1e7776b8e --- /dev/null +++ b/packages/server/src/workflows/runtime.test.ts @@ -0,0 +1,1605 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { describe, it } from "node:test" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { WorkflowDefinitionV1, WorkflowRun } from "../api-types" +import type { EventBus } from "../events/bus" +import type { Logger } from "../logger" +import type { WorkspaceManager } from "../workspaces/manager" +import { WorkflowInterpreter } from "./interpreter" +import { WorkflowManager } from "./manager" + +const workspaceManager = { + get: (id: string) => ({ id, lineageId: "lineage", path: "C:/workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }], +} as unknown as WorkspaceManager +const eventBus = { publish: () => true } as unknown as EventBus +const logger = { warn() {}, error() {} } as unknown as Logger +const execFileAsync = promisify(execFile) +const usage = (cost = 0.1, tokens = 10) => ({ + role: "assistant", cost, + tokens: { total: tokens, input: tokens, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, +}) +const workflowTools = { ids: async () => ({ data: ["read", "glob", "grep", "lsp", "bash", "shell", "write", "edit", "apply_patch", "task"] }) } +const waitFor = async (manager: WorkflowManager, id: string, statuses: WorkflowRun["status"][]) => { + let latest: WorkflowRun | undefined + for (let attempt = 0; attempt < 200; attempt += 1) { + latest = (await manager.get(id))! + if (statuses.includes(latest.status)) return latest + await new Promise((resolve) => setTimeout(resolve, 5)) + } + throw new Error(`Workflow ${id} did not reach ${statuses.join(", ")}: ${latest?.status} ${latest?.error ?? ""}`) +} + +describe("declarative workflow runtime", () => { + it("runs branch, foreach, parallel and bounded repeat with structured handoff", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-runtime-")) + const prompts: Array> = [] + let sessions = 0 + const client = { + tool: { ids: async () => ({ data: ["read", "shell", "write"] }) }, + session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async (input: Record) => { + prompts.push(input) + const text = JSON.stringify(input.parts) + return { data: text.includes("Seed") + ? { info: { ...usage(), structured: { go: true, items: [1, 2, 3] } }, parts: [] } + : { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + shell: async () => ({ data: { info: usage(), parts: [{ type: "tool", state: { status: "completed", output: "shell" } }] } }), + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + const definition: WorkflowDefinitionV1 = { + version: 1, id: "dynamic", name: "Dynamic", maxConcurrency: 4, + root: { type: "sequence", id: "root", steps: [ + { type: "agent", id: "seed", title: "Seed", instructions: "Seed", tools: ["read"], outputSchema: { + type: "object", required: ["go", "items"], properties: { go: { type: "boolean" }, items: { type: "array" } }, + } }, + { type: "condition", id: "branch", condition: { value: { $ref: "nodes.seed.output.go" }, equals: true }, then: { + type: "foreach", id: "each", items: { $ref: "nodes.seed.output.items" }, item: "item", maxItems: 3, maxConcurrency: 2, + body: { type: "agent", id: "worker", instructions: "Handle item", context: { $ref: "vars.item" } }, + } }, + { type: "parallel", id: "parallel", maxConcurrency: 2, branches: [ + { type: "agent", id: "left", instructions: "Left" }, + { type: "shell", id: "right", title: "Right", agent: "build", command: "git status --short" }, + ] }, + { type: "repeat", id: "repeat", maxIterations: 2, body: { type: "agent", id: "again", instructions: "Again" } }, + ] }, + } + try { + const stored = await manager.createDefinition(definition) + const started = await manager.start({ workspaceId: "workspace", definitionId: stored.id, objective: "Run graph" }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(run.definitionRevision, 1) + assert.deepEqual(run.definitionSnapshot, definition) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "worker").length, 3) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "again").length, 2) + assert.equal(run.executionNodes?.find((node) => node.definitionNodeId === "right")?.output, "shell") + assert.match(JSON.stringify(prompts), /Context.*1/) + assert.deepEqual((prompts[0]?.format as Record)?.type, "json_schema") + assert.deepEqual(prompts[0]?.tools, { "*": false, read: true, shell: false, write: false }) + assert.equal(run.usage?.tokens, 80) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps repeat references scoped to their foreach instance", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-repeat-scope-")) + let sessions = 0 + let seeds = 0 + let releaseSeeds!: () => void + const bothSeeds = new Promise((resolve) => { releaseSeeds = resolve }) + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `repeat-scope-${++sessions}` } }), + prompt: async (input: Record) => { + const prompt = JSON.stringify(input.parts) + if (prompt.includes("Seed")) { + if (++seeds === 2) releaseSeeds() + await bothSeeds + const item = prompt.includes("Context:\\n1") ? 1 : 0 + return { data: { info: { ...usage(), structured: item }, parts: [] } } + } + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "repeat-scope", name: "Repeat scope", maxConcurrency: 2, root: { + type: "foreach", id: "each", items: [0, 1], item: "item", maxItems: 2, maxConcurrency: 2, body: { + type: "sequence", id: "iteration", steps: [ + { type: "agent", id: "seed", title: "Seed", instructions: "Seed", context: { $ref: "vars.item" }, + outputSchema: { type: "number" } }, + { type: "repeat", id: "repeat", maxIterations: 1, + while: { value: { $ref: "nodes.seed.output" }, equals: { $ref: "vars.item" } }, + body: { type: "agent", id: "work", instructions: "Work" } }, + ], + }, + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "repeat-scope" }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "work").length, 2) + } finally { + releaseSeeds() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("allows shared context references but rejects ancestor cycles", () => { + const shared = { value: 1 } + const run = { + id: "context", workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Context", status: "running", steps: [], executionNodes: [], createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as WorkflowRun + const interpreter = new WorkflowInterpreter({ + run, client: {} as OpencodeClient, persist: async () => {}, signal: () => new AbortController().signal, + sessionStarted: () => true, sessionFinished: () => {}, abortSession: async () => true, isCancelled: () => false, + }) + const context = { vars: {}, inputs: { shared }, budgets: [], limiters: [], definitionInvocationKey: "context@1" } + const resolved = (interpreter as any).resolveContext( + [{ $ref: "inputs.shared" }, { $ref: "inputs.shared" }], context, new AbortController().signal, + ) + assert.equal(resolved[0], resolved[1]) + const cyclic: Record = {} + cyclic.self = cyclic + assert.throws(() => (interpreter as any).resolveContext( + { $ref: "inputs.cyclic" }, { ...context, inputs: { cyclic } }, new AbortController().signal, + ), /contains a cycle/) + }) + + it("disables omitted agent tools and rejects execution-capable tool requests", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-tools-")) + const prompts: Array> = [] + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `tools-${++sessions}` } }), + prompt: async (input: Record) => { + prompts.push(input) + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "tools-off", name: "Tools off", root: { + type: "agent", id: "work", instructions: "Read only", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "tools-off" }) + await waitFor(manager, started.id, ["completed"]) + assert.deepEqual(prompts[0]?.tools, { + "*": false, read: false, glob: false, grep: false, lsp: false, bash: false, + shell: false, write: false, edit: false, apply_patch: false, task: false, + }) + + await manager.createDefinition({ version: 1, id: "tools-dangerous", name: "Dangerous", root: { + type: "agent", id: "work", instructions: "Run", tools: ["bash"], + } }) + const dangerous = await manager.start({ workspaceId: "workspace", definitionId: "tools-dangerous" }) + const failed = await waitFor(manager, dangerous.id, ["failed"]) + assert.match(failed.error ?? "", /tool bash is not allowed/) + assert.equal(prompts.length, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("pauses at a durable boundary and resumes without repeating completed work", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pause-")) + let release!: () => void + const first = new Promise((resolve) => { release = resolve }) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async () => { prompts++; if (prompts === 1) await first; return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "pause", name: "Pause", root: { type: "sequence", id: "root", steps: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "pause" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + assert.equal((await manager.pause(started.id))?.status, "pausing") + release() + while (started.status !== "paused") await new Promise((resolve) => setTimeout(resolve, 1)) + const resumed = manager.resume(started.id) + const racedStart = manager.start({ workspaceId: "workspace", definitionId: "pause" }) + await resumed + await assert.rejects(racedStart, /already running/) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 2) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "one").length, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps interpreter node references attached when pause persistence fails", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pause-rollback-")) + let releaseFirst!: () => void + let releaseSecond!: () => void + const firstBlocked = new Promise((resolve) => { releaseFirst = resolve }) + const secondBlocked = new Promise((resolve) => { releaseSecond = resolve }) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `pause-rollback-${++sessions}` } }), + prompt: async () => { + prompts++ + await (prompts === 1 ? firstBlocked : secondBlocked) + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "pause-rollback", name: "Pause rollback", root: { + type: "sequence", id: "root", steps: [ + { type: "agent", id: "one", instructions: "One" }, + { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "pause-rollback" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + const persist = (manager as any).persist.bind(manager) + let pauseWriteStarted!: () => void + const pauseWrite = new Promise((resolve) => { pauseWriteStarted = resolve }) + let rejectPauseWrite!: () => void + const pauseWriteFailure = new Promise((_, reject) => { rejectPauseWrite = () => reject(new Error("pause write failed")) }) + let failed = false + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (!failed && run.status === "pausing") { + failed = true + pauseWriteStarted() + return pauseWriteFailure + } + return persist(run, touch) + } + const pausing = manager.pause(started.id) + await pauseWrite + releaseFirst() + while (prompts < 2 || started.usage?.tokens !== 10) await new Promise((resolve) => setTimeout(resolve, 1)) + rejectPauseWrite() + await assert.rejects(pausing, /pause write failed/) + assert.equal(started.status, "running") + assert.equal(started.pauseRequested, undefined) + assert.equal(started.usage?.tokens, 10) + assert.deepEqual(started.executionNodes?.filter((node) => node.type === "agent").map((node) => node.status), ["completed", "running"]) + releaseSecond() + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 2) + assert.deepEqual(run.executionNodes?.filter((node) => node.type === "agent").map((node) => node.status), ["completed", "completed"]) + } finally { + releaseFirst() + releaseSecond() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps a parallel pause pausing until every worker reaches a boundary", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-parallel-pause-")) + const releases: Array<() => void> = [] + const blocked = [0, 1].map(() => new Promise((resolve) => releases.push(resolve))) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `parallel-pause-${++sessions}` } }), + prompt: async () => { + const index = prompts++ + if (index < 2) await blocked[index] + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "parallel-pause", name: "Parallel pause", maxConcurrency: 2, + root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "sequence", id: "left", steps: [ + { type: "agent", id: "left-one", instructions: "One" }, + { type: "agent", id: "left-two", instructions: "Two" }, + ] }, + { type: "sequence", id: "right", steps: [ + { type: "agent", id: "right-one", instructions: "One" }, + { type: "agent", id: "right-two", instructions: "Two" }, + ] }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "parallel-pause" }) + while (prompts < 2) await new Promise((resolve) => setTimeout(resolve, 1)) + assert.equal((await manager.pause(started.id))?.status, "pausing") + releases[0]!() + while (!started.executionNodes?.some((node) => node.definitionNodeId.endsWith("-one") && node.status === "completed")) { + await new Promise((resolve) => setTimeout(resolve, 1)) + } + assert.equal(started.status, "pausing") + assert.equal(prompts, 2) + releases[1]!() + await waitFor(manager, started.id, ["paused"]) + assert.equal(prompts, 2) + await manager.resume(started.id) + await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 4) + } finally { + for (const release of releases) release() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("suspends a limiter waiter before it creates a session", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pause-waiter-")) + let release!: () => void + const blocked = new Promise((resolve) => { release = resolve }) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `pause-waiter-${++sessions}` } }), + prompt: async () => { prompts++; if (prompts === 1) await blocked; return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "pause-waiter", name: "Pause waiter", maxConcurrency: 2, + budget: { maxTokens: 100 }, root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "One" }, + { type: "agent", id: "two", instructions: "Two" }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "pause-waiter" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + await manager.pause(started.id) + release() + await waitFor(manager, started.id, ["paused"]) + assert.equal(prompts, 1) + assert.equal(sessions, 2) + await manager.resume(started.id) + await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 2) + } finally { + release() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("validates an input gate answer and stops after an observed budget overrun", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-gate-budget-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(2, 100), parts: [{ type: "text", text: "costly" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "gate", name: "Gate", budget: { maxCost: 1 }, root: { + type: "sequence", id: "root", steps: [ + { type: "gate", id: "input", gate: "input", prompt: "Name", inputSchema: { + type: "object", required: ["name"], properties: { name: { type: "string" } }, additionalProperties: false, + } }, + { type: "agent", id: "costly", instructions: "Spend" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "gate" }) + const waiting = await waitFor(manager, started.id, ["waiting_for_input"]) + const gateId = waiting.pendingGate!.executionNodeId + await assert.rejects(manager.answer(started.id, gateId, {}), /name is required/) + await manager.answer(started.id, gateId, { name: "Ada" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.match(run.error ?? "", /cost budget/) + assert.equal(run.usage?.cost, 2) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("fans cancellation out to every active parallel session", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-cancel-")) + let sessions = 0 + let prompts = 0 + const aborted = new Set() + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => { + prompts++ + return new Promise((_, reject) => options?.signal?.addEventListener("abort", () => reject(options.signal?.reason), { once: true })) + }, + abort: async ({ sessionID }: { sessionID: string }) => { aborted.add(sessionID); return { data: true } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "cancel", name: "Cancel", maxConcurrency: 2, root: { + type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "Wait" }, { type: "agent", id: "two", instructions: "Wait" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "cancel" }) + while (prompts < 2) await new Promise((resolve) => setTimeout(resolve, 1)) + await manager.cancel(started.id) + const run = await waitFor(manager, started.id, ["cancelled"]) + assert.equal(aborted.size, 2) + assert.equal(run.executionNodes?.filter((node) => node.status === "cancelled").length, 3) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("aborts a child session created after cancellation begins", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-create-cancel-")) + let sessions = 0 + let releaseCreate!: () => void + const childCreate = new Promise((resolve) => { releaseCreate = resolve }) + const aborted: string[] = [] + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => { + const id = `late-${++sessions}` + if (sessions === 2) await childCreate + return { data: { id } } + }, + prompt: async () => { prompts++; return { data: { info: usage(), parts: [] } } }, + abort: async ({ sessionID }: { sessionID: string }) => { aborted.push(sessionID); return { data: true } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "create-cancel", name: "Create cancel", root: { + type: "agent", id: "work", instructions: "Wait", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "create-cancel" }) + while (sessions < 2) await new Promise((resolve) => setTimeout(resolve, 1)) + const cancellation = manager.cancel(started.id) + releaseCreate() + assert.equal((await cancellation)?.status, "cancelled") + assert.deepEqual(aborted, ["late-2"]) + assert.equal(prompts, 0) + } finally { + releaseCreate() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("does not retry an unconfirmed side effect and retains recovery reservation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-ambiguous-")) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `ambiguous-${++sessions}` } }), + prompt: async () => { prompts++; throw new Error("connection lost") }, + abort: async () => ({ data: false }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "ambiguous", name: "Ambiguous", root: { + type: "agent", id: "work", instructions: "Act", retry: { maxAttempts: 2, idempotent: true }, + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "ambiguous" }) + const run = await waitFor(manager, started.id, ["recovery_required"]) + assert.equal(prompts, 1) + assert.equal(run.executionNodes?.find((node) => node.definitionNodeId === "work")?.status, "interrupted") + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "ambiguous" }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("serializes budgeted actions and enforces usage before admitting another action", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-budget-admission-")) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `budget-${++sessions}` } }), + prompt: async () => { prompts++; return { data: { info: usage(0.1, 11), parts: [{ type: "text", text: "spent" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "budget-admission", name: "Budget", maxConcurrency: 3, + budget: { maxTokens: 10 }, root: { type: "parallel", id: "root", maxConcurrency: 3, branches: [ + { type: "agent", id: "one", instructions: "One" }, + { type: "agent", id: "two", instructions: "Two" }, + { type: "agent", id: "three", instructions: "Three" }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "budget-admission" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.match(run.error ?? "", /token budget 10/) + assert.equal(prompts, 1) + assert.equal(run.usage?.tokens, 11) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("uses one action deadline across tool lookup, session creation and retries", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-deadline-")) + const signals: AbortSignal[] = [] + let sessions = 0 + let prompts = 0 + const client = { + tool: { ids: async (_input: unknown, options?: { signal?: AbortSignal }) => { + signals.push(options!.signal!); return { data: ["read"] } + } }, + session: { + create: async (_input: unknown, options?: { signal?: AbortSignal }) => { + sessions++ + if (sessions > 1) signals.push(options!.signal!) + return { data: { id: `deadline-${sessions}` } } + }, + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => { + prompts++ + signals.push(options!.signal!) + return new Promise((_, reject) => options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true })) + }, + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "deadline", name: "Deadline", root: { + type: "agent", id: "work", instructions: "Wait", timeoutMs: 20, retry: { maxAttempts: 2, idempotent: true }, + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "deadline" }) + await waitFor(manager, started.id, ["failed"]) + assert.equal(prompts, 1) + assert.equal(sessions, 2) + assert.ok(signals.every((signal) => signal === signals[0])) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("requires explicit recovery before an ambiguous persisted side effect can repeat", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-recovery-")) + const id = "00000000-0000-4000-8000-000000000099" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { + version: 1, id: "recover", name: "Recover", + root: { type: "agent", id: "work", instructions: "Potential side effect" }, + } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Recover", status: "running", rootSessionId: "root", steps: [], revision: 2, + definitionId: "recover", definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ + id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", status: "running", + attempt: 1, sessionIds: ["ambiguous"], startedAt: now, + }], + createdAt: now, updatedAt: now, + }), "utf8") + let sessions = 0 + const calls: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => { const id = `recovery-${++sessions}`; calls.push(`create:${id}`); return { data: { id } } }, + prompt: async () => { calls.push("prompt"); return { data: { info: usage(), parts: [{ type: "text", text: "confirmed" }] } } }, + abort: async ({ sessionID }: { sessionID: string }) => { calls.push(`abort:${sessionID}`); return { data: true } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + const recovered = (await manager.get(id))! + assert.equal(recovered.status, "recovery_required") + assert.equal(recovered.executionNodes?.[0]?.status, "interrupted") + await assert.rejects(manager.resume(id), /Recovery confirmation is required/) + await manager.resume(id, true) + const completed = await waitFor(manager, id, ["completed"]) + assert.equal(completed.executionNodes?.[0]?.output, "confirmed") + assert.deepEqual(calls.slice(0, 3), ["abort:ambiguous", "create:recovery-1", "prompt"]) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps recovery required when persisted session abort is unconfirmed", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-recovery-failed-")) + const id = "00000000-0000-4000-8000-000000000098" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { + version: 1, id: "recover-failed", name: "Recover failed", + root: { type: "agent", id: "work", instructions: "Potential side effect" }, + } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Recover", status: "recovery_required", rootSessionId: "root", steps: [], revision: 2, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "interrupted", attempt: 1, sessionIds: ["ambiguous"], startedAt: now, completedAt: now }], + createdAt: now, updatedAt: now, + }), "utf8") + let creates = 0 + const client = { tool: workflowTools, session: { + create: async () => { creates++; return { data: { id: "unexpected" } } }, + abort: async () => ({ data: false }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition(definition) + await assert.rejects(manager.resume(id, true), /could not confirm/) + const run = (await manager.get(id))! + assert.equal(run.status, "recovery_required") + assert.equal(run.executionNodes?.[0]?.status, "interrupted") + assert.equal(creates, 0) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: definition.id }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("recovers session-bearing paused nodes independent of node type before answering or cancelling", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-paused-session-")) + const id = "00000000-0000-4000-8000-000000000095" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "paused-session", name: "Paused", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Approve", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Paused", status: "paused", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "gate", definitionNodeId: "gate", type: "gate", + status: "waiting", attempt: 0, sessionIds: ["unexpected-session"], startedAt: now }], + pendingGate: { executionNodeId: "execution", definitionNodeId: "gate", gate: "approval", prompt: "Approve" }, + createdAt: now, updatedAt: now, + }), "utf8") + let confirmAbort = false + const aborted: string[] = [] + const client = { session: { + abort: async ({ sessionID }: { sessionID: string }) => { aborted.push(sessionID); return { data: confirmAbort } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + const recovered = (await manager.get(id))! + assert.equal(recovered.status, "recovery_required") + assert.equal(recovered.executionNodes?.[0]?.status, "interrupted") + await assert.rejects(manager.answer(id, "execution", true), /not waiting for a gate answer/) + assert.equal((await manager.cancel(id))?.status, "recovery_required") + confirmAbort = true + assert.equal((await manager.cancel(id))?.status, "cancelled") + assert.deepEqual(aborted, ["unexpected-session", "unexpected-session"]) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("serializes lineage admission and rejects a stale answer after the next gate opens", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-admission-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `gate-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "gates", name: "Gates", root: { + type: "sequence", id: "root", steps: [ + { type: "gate", id: "first", gate: "approval", prompt: "First" }, + { type: "gate", id: "second", gate: "approval", prompt: "Second" }, + ], + } }) + const starts = await Promise.allSettled([ + manager.start({ workspaceId: "workspace", definitionId: "gates" }), + manager.start({ workspaceId: "workspace", definitionId: "gates" }), + ]) + assert.equal(starts.filter((result) => result.status === "fulfilled").length, 1) + assert.match((starts.find((result) => result.status === "rejected") as PromiseRejectedResult).reason.message, /already running/) + const started = (starts.find((result) => result.status === "fulfilled") as PromiseFulfilledResult).value + const first = await waitFor(manager, started.id, ["waiting_for_review"]) + const firstGate = first.pendingGate!.executionNodeId + const answers = await Promise.allSettled([ + manager.answer(started.id, firstGate, true), + manager.answer(started.id, firstGate, true), + ]) + assert.equal(answers.filter((result) => result.status === "fulfilled").length, 1) + assert.match((answers.find((result) => result.status === "rejected") as PromiseRejectedResult).reason.message, /stale/) + const second = await waitFor(manager, started.id, ["waiting_for_review"]) + assert.equal(second.pendingGate?.definitionNodeId, "second") + await manager.approve(started.id, second.pendingGate!.executionNodeId) + await waitFor(manager, started.id, ["completed"]) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("pins and executes nested saved definitions with shared inputs, concurrency and budgets", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-nested-")) + let sessions = 0 + let prompts = 0 + let active = 0 + let maxActive = 0 + let release!: () => void + const blocked = new Promise((resolve) => { release = resolve }) + const seenPrompts: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `nested-${++sessions}` } }), + prompt: async (input: Record) => { + prompts++ + active++ + maxActive = Math.max(maxActive, active) + seenPrompts.push(JSON.stringify(input.parts)) + if (prompts === 1) await blocked + active-- + return { data: { info: usage(0.1, 10), parts: [{ type: "text", text: "nested-output" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ + version: 1, id: "child", name: "Child", budget: { maxTokens: 25 }, maxConcurrency: 1, + root: { type: "parallel", id: "child-root", branches: [ + { type: "agent", id: "child-work-1", instructions: "Old child instructions", context: { $ref: "inputs.message" } }, + { type: "agent", id: "child-work-2", instructions: "Old child instructions", context: { $ref: "inputs.message" } }, + ] }, + }) + await manager.createDefinition({ + version: 1, id: "parent", name: "Parent", maxConcurrency: 2, budget: { maxTokens: 15 }, + root: { type: "workflow", id: "nested", definitionId: "child", inputs: { message: { $ref: "inputs.message" } } }, + }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "parent", inputs: { message: "hello" } }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + await manager.updateDefinition("child", 1, { + version: 1, id: "child", name: "Child", + root: { type: "agent", id: "child-work", instructions: "New child instructions" }, + }) + release() + const run = await waitFor(manager, started.id, ["failed"]) + assert.equal(run.savedDefinitionSnapshots?.length, 1) + assert.equal(run.savedDefinitionSnapshots?.[0]?.revision, 1) + assert.equal((run.definitionSnapshot?.root as { definitionRevision?: number }).definitionRevision, 1) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId.startsWith("child-work-")).length, 2) + assert.equal(run.usage?.tokens, 20) + assert.match(run.error ?? "", /cost budget|token budget 15/) + assert.equal(maxActive, 1) + assert.match(seenPrompts.join("\n"), /Old child instructions/) + assert.match(seenPrompts.join("\n"), /hello/) + assert.doesNotMatch(seenPrompts.join("\n"), /New child instructions/) + } finally { + release() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("shares a saved-definition budget admission lock across parallel invocations", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-shared-nested-budget-")) + let sessions = 0 + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `shared-budget-${++sessions}` } }), + prompt: async () => { prompts++; return { data: { info: usage(0, 11), parts: [{ type: "text", text: "spent" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "shared-budget-child", name: "Child", budget: { maxTokens: 10 }, root: { + type: "agent", id: "spend", instructions: "Spend", + } }) + await manager.createDefinition({ version: 1, id: "shared-budget-parent", name: "Parent", maxConcurrency: 2, root: { + type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "workflow", id: "first", definitionId: "shared-budget-child" }, + { type: "workflow", id: "second", definitionId: "shared-budget-child" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "shared-budget-parent" }) + assert.match((await waitFor(manager, started.id, ["failed"])).error ?? "", /token budget 10/) + assert.equal(prompts, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps nodes references inside one saved-definition invocation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-reference-scope-")) + let sessions = 0 + const prompts: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `scope-${++sessions}` } }), + prompt: async (input: Record) => { + const prompt = JSON.stringify(input.parts) + prompts.push(prompt) + const text = prompt.includes("Parent seed") ? "parent-value" : prompt.includes("Child seed") ? "child-value" : "done" + return { data: { info: usage(), parts: [{ type: "text", text }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "scope-child", name: "Child", root: { + type: "sequence", id: "child-root", steps: [ + { type: "agent", id: "seed", instructions: "Child seed" }, + { type: "agent", id: "consume", instructions: "Child consume", context: { $ref: "nodes.seed.output" } }, + ], + } }) + await manager.createDefinition({ version: 1, id: "scope-parent", name: "Parent", root: { + type: "sequence", id: "parent-root", steps: [ + { type: "agent", id: "seed", instructions: "Parent seed" }, + { type: "workflow", id: "child", definitionId: "scope-child" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "scope-parent" }) + await waitFor(manager, started.id, ["completed"]) + const consume = prompts.find((prompt) => prompt.includes("Child consume")) ?? "" + assert.match(consume, /child-value/) + assert.doesNotMatch(consume, /parent-value/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("defaults each nested workflow invocation to one concurrent action", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-nested-default-")) + let sessions = 0 + let active = 0 + let maxActive = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `nested-default-${++sessions}` } }), + prompt: async () => { + active++ + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setTimeout(resolve, 5)) + active-- + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "nested-child", name: "Child", root: { + type: "parallel", id: "child-root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + await manager.createDefinition({ version: 1, id: "nested-parent", name: "Parent", maxConcurrency: 2, root: { + type: "workflow", id: "child", definitionId: "nested-child", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "nested-parent" }) + await waitFor(manager, started.id, ["completed"]) + assert.equal(maxActive, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("cancels nested limiter waiters without deadlocking an invalid transition or shutdown", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-limiter-cancel-")) + let sessions = 0 + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `limiter-${++sessions}` } }), + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => { + prompts++ + return new Promise((_, reject) => options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true })) + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "limiter-child", name: "Child", root: { + type: "parallel", id: "child-root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + await manager.createDefinition({ version: 1, id: "limiter-parent", name: "Parent", maxConcurrency: 2, root: { + type: "workflow", id: "child", definitionId: "limiter-child", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "limiter-parent" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + await assert.rejects(Promise.race([ + manager.resume(started.id), + new Promise((_, reject) => setTimeout(() => reject(new Error("resume deadlocked")), 100)), + ]), /cannot be resumed/) + await manager.shutdown() + assert.equal(prompts, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("serializes current-workspace rebinding against authoritative persisted state", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-bind-")) + const id = "00000000-0000-4000-8000-000000000097" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "bound", name: "Bound", root: { + type: "agent", id: "work", instructions: "Done", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "old", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Bound", status: "interrupted", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + worktreeSelection: { policy: { mode: "current" }, sourceWorkspaceId: "old", sourceWorkspaceLineageId: "lineage", + sourceWorkspacePath: "C:/workspace", workspaceId: "old", directory: "C:/workspace", created: false }, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "completed", attempt: 1, output: "done", startedAt: now, completedAt: now }], + createdAt: now, updatedAt: now, + }), "utf8") + const restoredWorkspaces = { + get: (workspaceId: string) => workspaceId === "restored" + ? { id: "restored", lineageId: "lineage", path: "C:/workspace", status: "ready" } + : undefined, + list: () => [{ id: "restored", lineageId: "lineage", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ workspaceManager: restoredWorkspaces, eventBus, logger, storageDir: directory }) + try { + const persist = (manager as any).persist.bind(manager) + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (run.workspaceId === "restored") throw new Error("bind write failed") + return persist(run, touch) + } + await assert.rejects(manager.get(id, "restored"), /bind write failed/) + assert.equal((await manager.get(id))?.workspaceId, "old") + assert.equal((manager as any).activeWorkspaces.get("old"), id) + assert.equal((manager as any).activeWorkspaces.has("restored"), false) + ;(manager as any).persist = persist + const [run, listed] = await Promise.all([manager.get(id, "restored"), manager.list("restored")]) + assert.equal(run?.workspaceId, "restored") + assert.equal(run?.worktreeSelection?.workspaceId, "restored") + assert.equal(run?.worktreeSelection?.sourceWorkspaceId, "restored") + assert.equal(listed[0]?.workspaceId, "restored") + const stored = JSON.parse(await fs.readFile(path.join(directory, `${id}.json`), "utf8")) as WorkflowRun + assert.equal(stored.worktreeSelection?.sourceWorkspaceId, "restored") + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("skips corrupt history and never prunes interrupted runs", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-history-")) + const now = new Date().toISOString() + for (let index = 0; index < 101; index++) await fs.writeFile(path.join(directory, `complete-${index}.json`), JSON.stringify({ + id: `complete-${index}`, workspaceId: `history-${index % 2}`, workspaceLineageId: `history-lineage-${index % 2}`, + workspacePath: `C:/history-${index % 2}`, + objective: "History", status: "completed", steps: [], createdAt: now, updatedAt: new Date(Date.now() + index).toISOString(), + }), "utf8") + await fs.writeFile(path.join(directory, "interrupted.json"), JSON.stringify({ + id: "interrupted", workspaceId: "interrupted-workspace", workspaceLineageId: "interrupted-lineage", workspacePath: "C:/interrupted", + objective: "Resume me", status: "interrupted", steps: [], createdAt: "9999-01-01T00:00:00.000Z", updatedAt: now, + }), "utf8") + await fs.writeFile(path.join(directory, "corrupt.json"), JSON.stringify({ id: "corrupt", status: "completed" }), "utf8") + let sessions = 0 + const client = { session: { + create: async () => ({ data: { id: `history-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "done" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + assert.equal((await manager.list()).some((run) => run.id === "interrupted"), true) + const started = await manager.start({ workspaceId: "workspace", objective: "Prune", stages: [{ id: "stage", title: "Stage", instructions: "Run" }] }) + await waitFor(manager, started.id, ["completed"]) + const entries = await fs.readdir(directory) + assert.equal(entries.includes("interrupted.json"), true) + assert.equal(entries.includes("corrupt.json"), true) + assert.equal(entries.filter((entry) => entry.endsWith(".json") && entry !== "interrupted.json" && entry !== "corrupt.json").length, 100) + await assert.doesNotReject(manager.list()) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("rejects saved workflow cycles and excessive nesting before creating sessions", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-nesting-")) + let sessions = 0 + const client = { tool: workflowTools, session: { create: async () => { sessions++; return { data: { id: "unexpected" } } } } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "cycle-a", name: "A", root: { type: "workflow", id: "call-b", definitionId: "cycle-b" } }) + await manager.createDefinition({ version: 1, id: "cycle-b", name: "B", root: { type: "workflow", id: "call-a", definitionId: "cycle-a" } }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "cycle-a" }), /cycle-a -> cycle-b -> cycle-a/) + + for (let index = 9; index >= 0; index -= 1) await manager.createDefinition({ + version: 1, id: `depth-${index}`, name: `Depth ${index}`, + root: index === 9 + ? { type: "condition", id: "leaf", condition: false, then: { type: "agent", id: "never", instructions: "Never" } } + : { type: "workflow", id: `call-${index + 1}`, definitionId: `depth-${index + 1}` }, + }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "depth-0" }), /maximum depth 8/) + assert.equal(sessions, 0) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("launches new and existing managed worktrees as retained OpenCode workspaces", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-worktree-run-")) + const repo = path.join(directory, "repo") + await fs.mkdir(repo) + await execFileAsync("git", ["init"], { cwd: repo }) + await fs.writeFile(path.join(repo, "README.md"), "initial\n") + await execFileAsync("git", ["add", "README.md"], { cwd: repo }) + await execFileAsync("git", ["-c", "user.name=CodeNomad", "-c", "user.email=test@example.com", "commit", "-m", "initial"], { cwd: repo }) + + const descriptors = new Map([["source", { + id: "source", lineageId: "source-lineage", path: repo, status: "ready", binaryId: "opencode", + }]]) + const launchedDirectories: string[] = [] + const cancelledCreationRequests: string[] = [] + let workspaceNumber = 0 + let retainCreation = true + let manager!: WorkflowManager + const managedWorkspaces = { + get: (id: string) => descriptors.get(id), + list: () => Array.from(descriptors.values()), + create: async (folder: string) => { + launchedDirectories.push(folder) + const existing = Array.from(descriptors.values()).find((entry) => path.resolve(entry.path) === path.resolve(folder)) + if (existing) return { workspace: existing, created: false } + const id = `target-${++workspaceNumber}` + const workspace = { id, lineageId: `${id}-lineage`, path: folder, status: "ready", binaryId: "opencode" } + descriptors.set(id, workspace) + return { workspace, created: true } + }, + releaseCreationRequest: () => retainCreation, + cancelCreationRequest: async (requestId: string) => { + cancelledCreationRequests.push(requestId) + assert.equal(await manager.withWorkspaceOwnershipLease({ id: `target-${workspaceNumber}` }, async (owned) => owned), false) + }, + } as unknown as WorkspaceManager + const clientWorkspaceIds: string[] = [] + const publishedWorkspaceIds: string[] = [] + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `worktree-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "done" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + manager = new WorkflowManager({ + workspaceManager: managedWorkspaces, + eventBus: { publish: (event: { instanceId: string }) => { publishedWorkspaceIds.push(event.instanceId); return true } } as unknown as EventBus, + logger, + storageDir: path.join(directory, "runs"), definitionsDir: path.join(directory, "definitions"), + createClient: (workspaceId) => { clientWorkspaceIds.push(workspaceId); return client }, + }) + const prunedLineages: string[] = [] + const pruneHistory = (manager as any).pruneHistory.bind(manager) + ;(manager as any).pruneHistory = async (lineageId: string) => { + prunedLineages.push(lineageId) + return pruneHistory(lineageId) + } + try { + await manager.createDefinition({ version: 1, id: "isolated", name: "Isolated", root: { type: "agent", id: "work", instructions: "Work" } }) + await assert.rejects(manager.start({ + workspaceId: "source", definitionId: "isolated", initiatorSessionId: "source-session", + worktree: { mode: "new", slug: "workflow-test" }, + }), /initiatorSessionId is unsupported/) + const created = await manager.start({ workspaceId: "source", definitionId: "isolated", worktree: { mode: "new", slug: "workflow-test" } }) + const first = await waitFor(manager, created.id, ["completed"]) + assert.equal(first.worktreeSelection?.created, true) + assert.equal(first.worktreeSelection?.slug, "workflow-test") + assert.equal(first.workspacePath, launchedDirectories[0]) + assert.equal(first.workspaceId, clientWorkspaceIds[0]) + assert.ok(publishedWorkspaceIds.includes("source")) + assert.ok(publishedWorkspaceIds.includes(first.workspaceId)) + await fs.writeFile(path.join(first.workspacePath, "dirty.txt"), "retain me\n") + + const reused = await manager.start({ workspaceId: "source", definitionId: "isolated", worktree: { mode: "existing", slug: "workflow-test" } }) + const second = await waitFor(manager, reused.id, ["completed"]) + assert.equal(second.worktreeSelection?.created, false) + assert.equal(second.workspacePath, first.workspacePath) + assert.deepEqual(prunedLineages, ["source-lineage", "source-lineage"]) + assert.equal(await fs.readFile(path.join(first.workspacePath, "dirty.txt"), "utf8"), "retain me\n") + retainCreation = false + await assert.rejects(manager.start({ workspaceId: "source", definitionId: "isolated", worktree: { + mode: "existing", slug: "workflow-test", + } }), /ownership could not be retained/) + assert.equal(cancelledCreationRequests.length, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("never repeats crash-interrupted actions without persisted termination evidence", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-no-recovery-evidence-")) + const id = "00000000-0000-4000-8000-000000000096" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "no-evidence", name: "No evidence", root: { + type: "agent", id: "work", instructions: "Do not repeat", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "No repeat", status: "running", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "running", attempt: 1, startedAt: now }], createdAt: now, updatedAt: now, + }), "utf8") + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: "unexpected" } }), + prompt: async () => { prompts++; return { data: { info: usage(), parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + assert.equal((await manager.get(id))?.status, "recovery_required") + await assert.rejects(manager.resume(id, true), /no persisted session IDs.*will not be repeated/) + assert.equal((await manager.get(id))?.status, "recovery_required") + assert.equal(prompts, 0) + await assert.rejects(manager.start({ workspaceId: "workspace", objective: "blocked", stages: [] }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("restores durable reservation state when cancel and resume persistence fail", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-transition-rollback-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `rollback-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "rollback", name: "Rollback", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Wait", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "rollback" }) + const waiting = await waitFor(manager, started.id, ["waiting_for_review"]) + const persist = (manager as any).persist.bind(manager) + let failCancel = true + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (failCancel && run.status === "cancelled") { failCancel = false; throw new Error("cancel write failed") } + return persist(run, touch) + } + await assert.rejects(manager.cancel(started.id), /cancel write failed/) + assert.equal((await manager.get(started.id))?.status, "waiting_for_review") + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "rollback" }), /already running/) + + let failResume = true + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (failResume && run.status === "running") { failResume = false; throw new Error("resume write failed") } + return persist(run, touch) + } + await assert.rejects(manager.answer(started.id, waiting.pendingGate!.executionNodeId, true), /resume write failed/) + const restored = await manager.get(started.id) + assert.equal(restored?.status, "waiting_for_review") + assert.equal(restored?.pendingGate?.executionNodeId, waiting.pendingGate!.executionNodeId) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "rollback" }), /already running/) + ;(manager as any).persist = persist + assert.equal((await manager.cancel(started.id))?.status, "cancelled") + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("blocks at exact observed budget and rejects malformed SDK usage", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-strict-budget-")) + let prompts = 0 + let sessions = 0 + let invalid = false + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `strict-${++sessions}` } }), + prompt: async () => { + prompts++ + return { data: { info: invalid ? usage(-1, Number.NaN) : usage(0, 10), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "exact-budget", name: "Exact", budget: { maxTokens: 10 }, root: { + type: "sequence", id: "root", steps: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + const exact = await manager.start({ workspaceId: "workspace", definitionId: "exact-budget" }) + const exactRun = await waitFor(manager, exact.id, ["failed"]) + assert.match(exactRun.error ?? "", /reached token budget 10/) + assert.equal(prompts, 1) + + invalid = true + await manager.createDefinition({ version: 1, id: "invalid-usage", name: "Invalid usage", root: { + type: "agent", id: "work", instructions: "Work", + } }) + const malformed = await manager.start({ workspaceId: "workspace", definitionId: "invalid-usage" }) + const malformedRun = await waitFor(manager, malformed.id, ["failed"]) + assert.match(malformedRun.error ?? "", /usage .* must be finite and non-negative/) + assert.equal(malformedRun.usage?.cost, 0) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("fails usage addition before a non-finite value can be persisted", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-usage-overflow-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `overflow-${++sessions}` } }), + prompt: async () => ({ data: { + info: { role: "assistant", cost: 0, tokens: { + input: Number.MAX_VALUE, output: Number.MAX_VALUE, reasoning: 0, cache: { read: 0, write: 0 }, + } }, + parts: [{ type: "text", text: "done" }], + } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "usage-overflow", name: "Usage overflow", root: { + type: "agent", id: "work", instructions: "Work", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "usage-overflow" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.match(run.error ?? "", /usage tokens.total overflowed/) + assert.equal(run.usage?.tokens, 0) + assert.doesNotMatch(await fs.readFile(path.join(directory, `${started.id}.json`), "utf8"), /"tokens": null/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("validates provider structured output and bounds resolved action context", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-output-context-")) + let prompts = 0 + let sessions = 0 + const seen: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `validation-${++sessions}` } }), + prompt: async (input: Record) => { + prompts++ + seen.push(JSON.stringify(input.parts)) + return { data: { info: { ...usage(), structured: { count: "wrong" } }, parts: [] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "schema-check", name: "Schema", root: { + type: "agent", id: "work", instructions: "Work", outputSchema: { + type: "object", required: ["count"], properties: { count: { type: "number" } }, + }, + } }) + const structured = await manager.start({ workspaceId: "workspace", definitionId: "schema-check" }) + assert.match((await waitFor(manager, structured.id, ["failed"])).error ?? "", /Structured output is invalid/) + + await manager.createDefinition({ version: 1, id: "context-check", name: "Context", root: { + type: "agent", id: "work", instructions: "Work", context: { $ref: "inputs.payload" }, + } }) + const oversized = await manager.start({ workspaceId: "workspace", definitionId: "context-check", inputs: { payload: "x".repeat(256_001) } }) + assert.match((await waitFor(manager, oversized.id, ["failed"])).error ?? "", /context exceeds 256000 bytes/) + await manager.createDefinition({ version: 1, id: "own-ref", name: "Own ref", root: { + type: "agent", id: "work", instructions: "Work", context: { $ref: "inputs.toString" }, + } }) + const inherited = await manager.start({ workspaceId: "workspace", definitionId: "own-ref", inputs: {} }) + await waitFor(manager, inherited.id, ["completed"]) + assert.equal(prompts, 2) + assert.doesNotMatch(seen.at(-1) ?? "", /Context:/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("allows bounded aggregate structural output above the action leaf limit", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-structural-output-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `aggregate-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "x".repeat(9_000) }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "aggregate-output", name: "Aggregate", maxConcurrency: 2, root: { + type: "parallel", id: "root", branches: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "aggregate-output" }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.ok(JSON.stringify(run.executionNodes?.find((node) => node.definitionNodeId === "root")?.output).length > 16_000) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("rejects composed expansion and aggregate saved graph bytes before effects", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-composed-limits-")) + let sessions = 0 + const client = { session: { create: async () => { sessions++; return { data: { id: "unexpected" } } } } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "expansion-child", name: "Child", root: { + type: "foreach", id: "each", items: [], item: "item", maxItems: 100, + body: { type: "agent", id: "work", instructions: "Work" }, + } }) + await manager.createDefinition({ version: 1, id: "expansion-parent", name: "Parent", root: { + type: "foreach", id: "outer", items: [], item: "item", maxItems: 100, + body: { type: "workflow", id: "child", definitionId: "expansion-child" }, + } }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "expansion-parent" }), /expand above limit/) + + const byteIds = ["bytes-a", "bytes-b", "bytes-c", "bytes-d", "bytes-e", "bytes-f"] + for (const id of byteIds) await manager.createDefinition({ version: 1, id, name: id, root: { + type: "agent", id: "work", instructions: "x".repeat(45_000), + } }) + await manager.createDefinition({ version: 1, id: "bytes-parent", name: "Bytes", root: { + type: "sequence", id: "root", steps: byteIds.map((id, index) => ({ + type: "workflow" as const, id: `call-${index}`, definitionId: id, + })), + } }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "bytes-parent" }), /graph exceeds 256000 bytes/) + assert.equal(sessions, 0) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("fails admission closed for malformed active records and quarantines identifiable ownership", async () => { + const blockedDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-malformed-global-")) + await fs.writeFile(path.join(blockedDir, "unknown.json"), "{not json", "utf8") + const blocked = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: blockedDir }) + try { + await assert.rejects(blocked.start({ workspaceId: "workspace", objective: "blocked", stages: [] }), /malformed active run unknown.json/) + assert.equal(await blocked.isWorkspaceWorkflowOwned({ lineageId: "anything" }), true) + } finally { + await blocked.shutdown() + await fs.rm(blockedDir, { recursive: true, force: true }) + } + + const quarantinedDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-malformed-lineage-")) + await fs.writeFile(path.join(quarantinedDir, "known.json"), JSON.stringify({ + id: "known", status: "running", workspaceId: "old", workspaceLineageId: "quarantined", workspacePath: "C:/quarantined", + }), "utf8") + const quarantined = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: quarantinedDir }) + try { + assert.equal(await quarantined.isWorkspaceWorkflowOwned({ lineageId: "quarantined" }), true) + assert.equal(await quarantined.isWorkspaceWorkflowOwned({ path: "C:/other" }), false) + } finally { + await quarantined.shutdown() + await fs.rm(quarantinedDir, { recursive: true, force: true }) + } + }) + + it("rejects persisted execution types that disagree with the pinned graph", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pinned-node-type-")) + const id = "wrong-type" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "typed", name: "Typed", root: { + type: "agent", id: "work", instructions: "Work", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "typed-lineage", workspacePath: "C:/typed", + objective: "Typed", status: "paused", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "sequence", + status: "waiting", attempt: 0 }], createdAt: now, updatedAt: now, + }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory }) + try { + await assert.rejects(manager.get(id), /pinned graph/) + assert.equal(await manager.isWorkspaceWorkflowOwned({ lineageId: "typed-lineage" }), true) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("exposes uncapped canonical ownership leases for retained execution worktrees", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-ownership-")) + const now = new Date().toISOString() + for (let index = 0; index < 101; index++) await fs.writeFile(path.join(directory, `done-${index}.json`), JSON.stringify({ + id: `done-${index}`, workspaceId: "old", workspaceLineageId: "old", workspacePath: "C:/old", + objective: "Done", status: "completed", steps: [], createdAt: now, updatedAt: new Date(Date.now() + index).toISOString(), + }), "utf8") + await fs.writeFile(path.join(directory, "owned.json"), JSON.stringify({ + id: "owned", workspaceId: "execution", workspaceLineageId: "execution-lineage", workspacePath: "C:/repo/.worktrees/job", + objective: "Owned", status: "interrupted", steps: [], + worktreeSelection: { policy: { mode: "existing", slug: "job" }, sourceWorkspaceId: "source-old", + sourceWorkspaceLineageId: "source-lineage", sourceWorkspacePath: "C:/repo", workspaceId: "execution", + directory: "C:/repo/.worktrees/job", slug: "job", created: false }, + createdAt: "2000-01-01T00:00:00.000Z", updatedAt: now, + }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory }) + try { + assert.equal(await manager.isWorkspaceWorkflowOwned({ lineageId: "source-lineage", path: "C:/repo" }), true) + assert.equal(await manager.isWorkspaceWorkflowOwned({ lineageId: "execution-lineage", path: "C:/repo/.worktrees/job" }), true) + assert.equal(await manager.isWorktreeWorkflowOwned({ lineageId: "source-lineage" }, { slug: "job" }), true) + assert.equal(await manager.isWorktreeWorkflowOwned({ lineageId: "wrong-source" }, { path: "C:/repo/.worktrees/job" }), true) + const leased = await manager.withWorktreeOwnershipLease( + { path: "C:/repo" }, { path: "C:/repo/.worktrees/job" }, async (owned) => owned, + ) + assert.equal(leased, true) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("prepares gate execution before consuming it and supports atomic owned cancellation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-gate-prepare-")) + let ready = true + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `gate-prepare-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const ownedWorkspaces = { + get: (id: string) => id === "wrong-workspace" + ? { id, lineageId: "wrong-lineage", path: "C:/wrong", status: "ready" } + : { id, lineageId: "lineage", path: "C:/workspace", status: "ready" }, + list: () => [{ id: "restored", lineageId: "lineage", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ workspaceManager: ownedWorkspaces, eventBus, logger, storageDir: directory, createClient: () => ready ? client : null }) + try { + await manager.createDefinition({ version: 1, id: "gate-prepare", name: "Gate", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Approve", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "gate-prepare" }) + const waiting = await waitFor(manager, started.id, ["waiting_for_review"]) + ready = false + await assert.rejects(manager.answer(started.id, waiting.pendingGate!.executionNodeId, true), /not ready/) + assert.equal((await manager.get(started.id))?.pendingGate?.executionNodeId, waiting.pendingGate!.executionNodeId) + assert.equal(await manager.cancelOwned(started.id, "wrong-workspace"), undefined) + const cancelled = await manager.cancelOwned(started.id, "restored") + assert.equal(cancelled?.status, "cancelled") + assert.equal(cancelled?.workspaceId, "restored") + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("uses the persisted bounded legacy output for the next-stage handoff", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-legacy-handoff-")) + const tail = "UNBOUNDED-TAIL" + const prompts: string[] = [] + let sessions = 0 + const client = { session: { + create: async () => ({ data: { id: `legacy-handoff-${++sessions}` } }), + prompt: async (input: Record) => { + prompts.push(JSON.stringify(input.parts)) + return { data: { info: {}, parts: [{ type: "text", text: prompts.length === 1 ? `${"x".repeat(20_000)}${tail}` : "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + const started = await manager.start({ workspaceId: "workspace", objective: "Bound handoff", stages: [ + { id: "one", title: "One", instructions: "One" }, + { id: "two", title: "Two", instructions: "Two" }, + ] }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal((run.steps[0]?.output as string).length, 16_000) + assert.equal(run.steps[0]?.outputTruncated, true) + assert.doesNotMatch(prompts[1] ?? "", new RegExp(tail)) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("clears a pending gate when a parallel branch fails", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-gate-branch-failure-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `gate-failure-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "gate-branch-failure", name: "Gate branch failure", maxConcurrency: 2, + root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "gate", id: "gate", gate: "approval", prompt: "Wait" }, + { type: "agent", id: "fail", instructions: "Fail", tools: ["bash"] }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "gate-branch-failure" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.equal(run.pendingGate, undefined) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("retains the reservation when terminal and fallback persistence both fail", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-terminal-persist-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `terminal-persist-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "done" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + let reloaded: WorkflowManager | undefined + try { + await manager.createDefinition({ version: 1, id: "terminal-persist", name: "Terminal persist", root: { + type: "agent", id: "work", instructions: "Work", + } }) + const persist = (manager as any).persist.bind(manager) + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (run.status === "completed" || run.status === "failed") throw new Error("terminal write failed") + return persist(run, touch) + } + const started = await manager.start({ workspaceId: "workspace", definitionId: "terminal-persist" }) + for (let attempt = 0; attempt < 200 && !(manager as any).activeRuns.get(started.id)?.releaseBlocked; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + } + assert.equal((manager as any).activeRuns.get(started.id)?.releaseBlocked, true) + const recovery = await manager.get(started.id) + assert.equal(recovery?.status, "recovery_required") + assert.equal(recovery?.pendingGate, undefined) + assert.equal(await fs.stat(path.join(directory, `${started.id}.recovery`)).then(() => true), true) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "terminal-persist" }), /already running/) + ;(manager as any).persist = persist + reloaded = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + assert.equal((await reloaded.get(started.id))?.status, "recovery_required") + await assert.rejects(reloaded.start({ workspaceId: "workspace", definitionId: "terminal-persist" }), /already running/) + } finally { + await manager.shutdown() + await reloaded?.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("uses a recovery marker instead of admitting an older waiting snapshot", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-recovery-marker-")) + const id = "00000000-0000-4000-8000-000000000094" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "marked-gate", name: "Marked gate", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Do not approve", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Marked", status: "waiting_for_review", steps: [], revision: 2, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "gate", definitionNodeId: "gate", type: "gate", + status: "waiting", attempt: 0, startedAt: now }], + pendingGate: { executionNodeId: "execution", definitionNodeId: "gate", gate: "approval", prompt: "Do not approve" }, + createdAt: now, updatedAt: now, + }), "utf8") + await fs.writeFile(path.join(directory, `${id}.recovery`), JSON.stringify({ runId: id }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory }) + try { + const run = await manager.get(id) + assert.equal(run?.status, "recovery_required") + assert.equal(run?.pendingGate, undefined) + await assert.rejects(manager.approve(id, "execution"), /not waiting for approval/) + await assert.rejects(manager.start({ workspaceId: "workspace", objective: "blocked", stages: [] }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workspaces/__tests__/git-worktrees.test.ts b/packages/server/src/workspaces/__tests__/git-worktrees.test.ts index bc6382a1c..321f2d0ec 100644 --- a/packages/server/src/workspaces/__tests__/git-worktrees.test.ts +++ b/packages/server/src/workspaces/__tests__/git-worktrees.test.ts @@ -1,9 +1,42 @@ import assert from "node:assert/strict" -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import path from "node:path" import { describe, it } from "node:test" -import { listWorktrees } from "../git-worktrees" +import { createManagedWorktree, listWorktrees, resolveRepoRoot } from "../git-worktrees" + +async function waitForFile(file: string): Promise { + const deadline = Date.now() + 2_000 + while (!existsSync(file) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)) + } + assert.equal(existsSync(file), true, `Timed out waiting for ${file}`) +} + +function installHangingGit(temp: string): { binDir: string; started: string; survived: string } { + const binDir = path.join(temp, "bin") + const script = path.join(temp, "hanging-git.cjs") + const started = path.join(temp, "started") + const survived = path.join(temp, "descendant-survived") + mkdirSync(binDir, { recursive: true }) + writeFileSync(script, [ + 'const { spawn } = require("node:child_process")', + 'const { writeFileSync } = require("node:fs")', + `writeFileSync(${JSON.stringify(started)}, "started")`, + "process.on(\"SIGTERM\", () => {})", + `spawn(process.execPath, ["-e", ${JSON.stringify(`process.on("SIGTERM", () => {}); setTimeout(() => require("node:fs").writeFileSync(${JSON.stringify(survived)}, "survived"), 700); setInterval(() => {}, 10_000)`) }], { stdio: "ignore" })`, + "setInterval(() => {}, 10_000)", + ].join("\n")) + + const gitPath = path.join(binDir, process.platform === "win32" ? "git.cmd" : "git") + if (process.platform === "win32") { + writeFileSync(gitPath, `@echo off\r\n"${process.execPath}" "${script}" %*\r\n`) + } else { + const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'` + writeFileSync(gitPath, `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(script)} "$@"\n`, { mode: 0o755 }) + } + return { binDir, started, survived } +} describe("listWorktrees", () => { it("uses the selected workspace folder for the root worktree directory", async () => { @@ -46,3 +79,53 @@ describe("listWorktrees", () => { } }) }) + +describe("resolveRepoRoot cancellation", () => { + for (const boundary of ["abort", "timeout"] as const) { + it(`terminates the complete git process tree on ${boundary} and waits for close`, async () => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-git-cancel-")) + const originalPath = process.env.PATH + const { binDir, started, survived } = installHangingGit(temp) + const controller = new AbortController() + + try { + process.env.PATH = `${binDir}${path.delimiter}${originalPath ?? ""}` + const operation = resolveRepoRoot(temp, undefined, { + signal: controller.signal, + timeoutMs: boundary === "timeout" ? 40 : 5_000, + }) + await waitForFile(started) + if (boundary === "abort") controller.abort(new Error("abort requested")) + + await assert.rejects(operation, boundary === "abort" ? /abort requested/ : /timed out after 40ms/) + await new Promise((resolve) => setTimeout(resolve, 850)) + assert.equal(existsSync(survived), false, "git helper descendant continued after cancellation completed") + } finally { + process.env.PATH = originalPath + rmSync(temp, { recursive: true, force: true }) + } + }) + } +}) + +describe("createManagedWorktree", () => { + it("rejects a managed worktree root that escapes through a symlink or junction", async () => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-managed-worktree-")) + const repoRoot = path.join(temp, "repo") + const outside = path.join(temp, "outside") + + try { + mkdirSync(path.join(repoRoot, ".codenomad"), { recursive: true }) + mkdirSync(outside, { recursive: true }) + symlinkSync(outside, path.join(repoRoot, ".codenomad", "worktrees"), process.platform === "win32" ? "junction" : "dir") + + await assert.rejects( + createManagedWorktree({ repoRoot, workspaceFolder: repoRoot, slug: "escaped" }), + /escapes repository/, + ) + assert.equal(existsSync(path.join(outside, "escaped")), false) + } finally { + rmSync(temp, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workspaces/__tests__/workspace-identity.test.ts b/packages/server/src/workspaces/__tests__/workspace-identity.test.ts index ccf8e6ca4..59e569b2f 100644 --- a/packages/server/src/workspaces/__tests__/workspace-identity.test.ts +++ b/packages/server/src/workspaces/__tests__/workspace-identity.test.ts @@ -183,4 +183,17 @@ describe("workspace identity", () => { assert.equal(reused.created, false) assert.equal(reused.workspace.id, normal.workspace.id) }) + + it("uses an explicit lineage before reusing a workspace by path", async () => { + const { root, target, link } = await createLinkedWorkspace() + const manager = createManager(root) + const normal = await manager.create(target) + const restored = await manager.create(link, undefined, { lineageId: "restored-lineage" }) + const repeated = await manager.create(target, undefined, { lineageId: "restored-lineage" }) + + assert.notEqual(restored.workspace.id, normal.workspace.id) + assert.equal(restored.workspace.lineageId, "restored-lineage") + assert.equal(repeated.workspace.id, restored.workspace.id) + assert.equal(repeated.created, false) + }) }) diff --git a/packages/server/src/workspaces/git-worktrees.ts b/packages/server/src/workspaces/git-worktrees.ts index 087009015..f951f22f4 100644 --- a/packages/server/src/workspaces/git-worktrees.ts +++ b/packages/server/src/workspaces/git-worktrees.ts @@ -1,7 +1,22 @@ import path from "path" -import { spawn } from "child_process" +import { spawn, spawnSync, type ChildProcess } from "child_process" +import { randomBytes } from "node:crypto" import type { WorktreeDescriptor } from "../api-types" import { promises as fsp } from "fs" +import { buildSpawnSpec } from "./spawn" +import { + LAUNCH_CLEANUP_TOKEN_ENV, + probeLaunchCleanupToken, + probePosixProcesses, + signalLaunchCleanupToken, + signalOwnedPosixProcessGroup, + signalPosixProcesses, +} from "./process-identity" + +const DEFAULT_GIT_TIMEOUT_MS = 30_000 +const GIT_GRACEFUL_STOP_MS = 250 +const GIT_CLEANUP_TIMEOUT_MS = 2_000 +const GIT_CLEANUP_COMMAND_TIMEOUT_MS = 500 export interface LogLike { debug?: (obj: any, msg?: string) => void @@ -9,16 +24,158 @@ export interface LogLike { } type GitResult = { ok: true; stdout: string } | { ok: false; error: Error; stdout?: string; stderr?: string } +type GitRunOptions = { signal?: AbortSignal; timeoutMs?: number } function isGitUnavailableResult(result: GitResult): boolean { return !result.ok && (result.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT" } -function runGit(args: string[], cwd: string): Promise { - return new Promise((resolve) => { - const child = spawn("git", args, { cwd, stdio: ["ignore", "pipe", "pipe"] }) +async function terminateGitProcessTree(child: ChildProcess, cleanupToken: string): Promise { + const pid = child.pid + if (!pid) throw new Error("spawned git process did not expose a PID") + + const deadline = Date.now() + GIT_CLEANUP_TIMEOUT_MS + const failures: string[] = [] + let closed = child.exitCode !== null || child.signalCode !== null + const onClose = () => { closed = true } + child.once("close", onClose) + + const remainingCommandTime = () => Math.max(1, Math.min(GIT_CLEANUP_COMMAND_TIMEOUT_MS, deadline - Date.now())) + const waitUntil = async (condition: () => boolean, until: number): Promise => { + while (Date.now() < until) { + if (condition()) return true + await new Promise((resolve) => setTimeout(resolve, 20)) + } + return condition() + } + + try { + if (process.platform === "win32") { + let treeCleanupConfirmed = false + const stopTree = (force: boolean) => { + if (Date.now() >= deadline) return + try { + const result = spawnSync( + "taskkill.exe", + ["/PID", String(pid), "/T", ...(force ? ["/F"] : [])], + { encoding: "utf8", timeout: remainingCommandTime() }, + ) + if (result.status === 0) treeCleanupConfirmed = true + else failures.push(String(result.stderr || result.stdout || result.error?.message || `exit code ${result.status}`).trim()) + } catch (error) { + failures.push(error instanceof Error ? error.message : String(error)) + } + } + + // Windows wrappers can exit before their descendants after a non-forced taskkill. + stopTree(true) + if (!await waitUntil(() => closed && treeCleanupConfirmed, deadline)) { + throw new Error(`git Windows process-tree cleanup was not confirmed before the cleanup deadline${failures.length ? `: ${failures.join("; ")}` : ""}`) + } + return + } + + const signalTree = (signal: NodeJS.Signals) => { + if (Date.now() >= deadline) return + const snapshot = probePosixProcesses(spawnSync, remainingCommandTime(), process.platform, { groupId: pid }) + const members = snapshot.ok + ? [...snapshot.processes.values()].filter((identity) => identity.groupId === pid) + : [] + const leader = members.find((identity) => identity.pid === pid) ?? members[0] + const result = leader + ? signalPosixProcesses(spawnSync, { + leader, + groupId: pid, + members, + signal, + allowLeaderlessGroup: true, + cleanupToken, + }, remainingCommandTime(), process.platform) + : signalOwnedPosixProcessGroup(spawnSync, pid, signal, remainingCommandTime()) + if (!result.ok) failures.push(`POSIX ${signal}: ${result.error}`) + if (process.platform === "linux") { + const tokenResult = signalLaunchCleanupToken(spawnSync, cleanupToken, signal, remainingCommandTime()) + if (!tokenResult.ok) failures.push(`launch-token ${signal}: ${tokenResult.error}`) + } + } + const treeGone = () => { + const snapshot = probePosixProcesses(spawnSync, remainingCommandTime(), process.platform, { groupId: pid }) + if (!snapshot.ok || [...snapshot.processes.values()].some((identity) => identity.groupId === pid)) return false + if (process.platform !== "linux") return true + const tokenSnapshot = probeLaunchCleanupToken(spawnSync, cleanupToken, remainingCommandTime()) + return tokenSnapshot.ok && tokenSnapshot.processes.size === 0 + } + + signalTree("SIGTERM") + if (!await waitUntil(() => closed && treeGone(), Math.min(deadline, Date.now() + GIT_GRACEFUL_STOP_MS))) { + signalTree("SIGKILL") + } + if (!await waitUntil(() => closed && treeGone(), deadline)) { + throw new Error(`git POSIX process-tree cleanup was not confirmed before the cleanup deadline${failures.length ? `: ${failures.join("; ")}` : ""}`) + } + } finally { + child.removeListener("close", onClose) + } +} + +function runGit(args: string[], cwd: string, options: GitRunOptions = {}): Promise { + return new Promise((resolve, reject) => { + options.signal?.throwIfAborted() + const cleanupToken = randomBytes(32).toString("hex") + const spec = buildSpawnSpec("git", args, { + cwd, + env: { ...process.env, [LAUNCH_CLEANUP_TOKEN_ENV]: cleanupToken }, + }) + const child = spawn(spec.command, spec.args, { + cwd: spec.cwd, + env: spec.env, + stdio: ["ignore", "pipe", "pipe"], + detached: process.platform !== "win32", + windowsVerbatimArguments: Boolean(spec.options.windowsVerbatimArguments), + }) let stdout = "" let stderr = "" + let settled = false + let terminating = false + let timeout: ReturnType | undefined + const requestedTimeout = options.timeoutMs ?? DEFAULT_GIT_TIMEOUT_MS + const timeoutMs = Number.isFinite(requestedTimeout) ? Math.max(1, requestedTimeout) : DEFAULT_GIT_TIMEOUT_MS + + const cleanup = () => { + if (timeout) clearTimeout(timeout) + options.signal?.removeEventListener("abort", abort) + } + const finish = (result: GitResult) => { + if (settled) return + settled = true + cleanup() + resolve(result) + } + const terminate = (error: unknown) => { + if (settled || terminating) return + terminating = true + cleanup() + void terminateGitProcessTree(child, cleanupToken).then( + () => { + settled = true + reject(error) + }, + (cleanupError) => { + settled = true + const reason = error instanceof Error ? error : new Error(String(error)) + const combined = new Error(`${reason.message}; ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`) + ;(combined as Error & { cause?: unknown }).cause = new AggregateError([reason, cleanupError]) + reject(combined) + }, + ) + } + const abort = () => terminate(options.signal?.reason ?? new Error("Git operation aborted")) + timeout = setTimeout( + () => terminate(new Error(`git ${args.join(" ")} timed out after ${timeoutMs}ms`)), + timeoutMs, + ) + options.signal?.addEventListener("abort", abort, { once: true }) + if (options.signal?.aborted) abort() child.stdout?.on("data", (chunk) => { stdout += chunk.toString() @@ -27,21 +184,27 @@ function runGit(args: string[], cwd: string): Promise { stderr += chunk.toString() }) child.once("error", (error) => { - resolve({ ok: false, error, stdout, stderr }) + if (terminating) return + finish({ ok: false, error, stdout, stderr }) }) child.once("close", (code) => { + if (terminating) return if (code === 0) { - resolve({ ok: true, stdout }) + finish({ ok: true, stdout }) } else { const error = new Error(stderr.trim() || `git ${args.join(" ")} failed with code ${code}`) - resolve({ ok: false, error, stdout, stderr }) + finish({ ok: false, error, stdout, stderr }) } }) }) } -export async function resolveRepoRoot(folder: string, logger?: LogLike): Promise<{ repoRoot: string; isGitRepo: boolean }> { - const result = await runGit(["rev-parse", "--show-toplevel"], folder) +export async function resolveRepoRoot( + folder: string, + logger?: LogLike, + options: GitRunOptions = {}, +): Promise<{ repoRoot: string; isGitRepo: boolean }> { + const result = await runGit(["rev-parse", "--show-toplevel"], folder, options) if (isGitUnavailableResult(result)) { throw new Error("Git is not installed or not available in PATH") } @@ -100,10 +263,12 @@ export async function listWorktrees(params: { repoRoot: string workspaceFolder: string logger?: LogLike + signal?: AbortSignal + timeoutMs?: number }): Promise { const { repoRoot, workspaceFolder, logger } = params - const result = await runGit(["worktree", "list", "--porcelain"], workspaceFolder) + const result = await runGit(["worktree", "list", "--porcelain"], workspaceFolder, params) if (!result.ok) { const rootDescriptor: WorktreeDescriptor = { slug: "root", directory: workspaceFolder, kind: "root" } logger?.debug?.({ repoRoot, err: result.error }, "Failed to list git worktrees; returning root only") @@ -170,11 +335,35 @@ export function isValidWorktreeSlug(slug: string): boolean { return true } +export async function isManagedWorktree(params: { + repoRoot: string + worktree: WorktreeDescriptor +}): Promise { + if (params.worktree.kind !== "worktree") return false + try { + const repoRoot = await fsp.realpath(params.repoRoot) + const managedRoot = await fsp.realpath(path.join(params.repoRoot, ".codenomad", "worktrees")) + const directory = await fsp.realpath(params.worktree.directory) + const normalize = (value: string) => process.platform === "win32" ? value.toLowerCase() : value + if (normalize(managedRoot) !== normalize(path.join(repoRoot, ".codenomad", "worktrees"))) return false + if (normalize(path.dirname(directory)) !== normalize(managedRoot)) return false + + const metadata = await fsp.readFile(path.join(directory, ".git"), "utf8") + const gitDir = metadata.match(/^gitdir:\s*(.+)$/m)?.[1]?.trim() + if (!gitDir) return false + return (await fsp.stat(path.resolve(directory, gitDir))).isDirectory() + } catch { + return false + } +} + export async function createManagedWorktree(params: { repoRoot: string workspaceFolder: string slug: string logger?: LogLike + signal?: AbortSignal + timeoutMs?: number }): Promise<{ slug: string; directory: string; branch?: string }> { const { repoRoot, workspaceFolder, logger } = params const branch = params.slug.trim() @@ -194,15 +383,32 @@ export async function createManagedWorktree(params: { return normalized || "worktree" } - const worktreesDir = path.join(repoRoot, ".codenomad", "worktrees") - const targetDir = path.join(worktreesDir, sanitizeDirName(branch)) + const canonicalRepoRoot = await fsp.realpath(repoRoot) + const codenomadDir = path.join(repoRoot, ".codenomad") + const worktreesDir = path.join(codenomadDir, "worktrees") + const pathsEqual = (left: string, right: string) => process.platform === "win32" + ? left.toLowerCase() === right.toLowerCase() + : left === right + + await fsp.mkdir(codenomadDir, { recursive: true }) + const canonicalCodenomadDir = await fsp.realpath(codenomadDir) + if (!pathsEqual(canonicalCodenomadDir, path.join(canonicalRepoRoot, ".codenomad"))) { + throw new Error("Managed worktree directory escapes repository") + } await fsp.mkdir(worktreesDir, { recursive: true }) + const canonicalWorktreesDir = await fsp.realpath(worktreesDir) + if (!pathsEqual(canonicalWorktreesDir, path.join(canonicalCodenomadDir, "worktrees"))) { + throw new Error("Managed worktree directory escapes repository") + } + const targetDir = path.join(canonicalWorktreesDir, sanitizeDirName(branch)) + const relativeTarget = path.relative(canonicalWorktreesDir, targetDir) + if (!relativeTarget || relativeTarget.startsWith(`..${path.sep}`) || path.isAbsolute(relativeTarget)) { + throw new Error("Managed worktree target escapes managed directory") + } try { - const stat = await fsp.stat(targetDir) - if (stat.isDirectory()) { - throw new Error("Worktree directory already exists") - } + await fsp.lstat(targetDir) + throw new Error("Worktree directory already exists") } catch (error) { const code = (error as NodeJS.ErrnoException).code if (code !== "ENOENT") { @@ -213,7 +419,7 @@ export async function createManagedWorktree(params: { logger?.debug?.({ slug: branch, branch, targetDir }, "Creating managed git worktree") // Prefer creating a new branch from HEAD. - const first = await runGit(["worktree", "add", "-b", branch, targetDir, "HEAD"], workspaceFolder) + const first = await runGit(["worktree", "add", "-b", branch, targetDir, "HEAD"], workspaceFolder, params) if (first.ok) { return { slug: branch, directory: targetDir, branch } } @@ -221,7 +427,7 @@ export async function createManagedWorktree(params: { const message = first.stderr?.toLowerCase() ?? first.error.message.toLowerCase() if (message.includes("already exists")) { // If the branch already exists, add worktree for that branch. - const second = await runGit(["worktree", "add", targetDir, branch], workspaceFolder) + const second = await runGit(["worktree", "add", targetDir, branch], workspaceFolder, params) if (second.ok) { return { slug: branch, directory: targetDir, branch } } diff --git a/packages/server/src/workspaces/instance-client.test.ts b/packages/server/src/workspaces/instance-client.test.ts new file mode 100644 index 000000000..92e0489fc --- /dev/null +++ b/packages/server/src/workspaces/instance-client.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import type { WorkspaceManager } from "./manager" +import { createInstanceClient } from "./instance-client" + +const workspaceManager = { + getInstancePort: () => 4321, + getInstanceAuthorizationHeader: () => undefined, + get: () => ({ id: "workspace", path: "C:/workspace", status: "ready" }), +} as unknown as WorkspaceManager + +describe("instance client fetch cancellation", () => { + it("preserves the SDK Request signal while retaining the fallback timeout", async () => { + const originalFetch = globalThis.fetch + let effectiveSignal: AbortSignal | null | undefined + globalThis.fetch = ((_input, init) => { + effectiveSignal = init?.signal + return new Promise((_resolve, reject) => effectiveSignal?.addEventListener("abort", () => reject(effectiveSignal?.reason), { once: true })) + }) as typeof fetch + try { + const client = createInstanceClient(workspaceManager, "workspace", { timeoutMs: 1_000 })! + const controller = new AbortController() + const request = client.tool.ids(undefined, { signal: controller.signal }) + while (!effectiveSignal) await new Promise((resolve) => setImmediate(resolve)) + const reason = new Error("caller cancelled") + controller.abort(reason) + assert.equal(effectiveSignal.aborted, true) + assert.strictEqual(effectiveSignal.reason, reason) + await request.catch(() => undefined) + } finally { + globalThis.fetch = originalFetch + } + }) + + it("times out requests even when the Request carries its default signal", async () => { + const originalFetch = globalThis.fetch + let effectiveSignal: AbortSignal | null | undefined + globalThis.fetch = ((_input, init) => { + effectiveSignal = init?.signal + return new Promise((_resolve, reject) => effectiveSignal?.addEventListener("abort", () => reject(effectiveSignal?.reason), { once: true })) + }) as typeof fetch + try { + const request = createInstanceClient(workspaceManager, "workspace", { timeoutMs: 5 })!.tool.ids() + while (!effectiveSignal?.aborted) await new Promise((resolve) => setTimeout(resolve, 1)) + assert.equal(effectiveSignal.reason?.name, "TimeoutError") + await request.catch(() => undefined) + } finally { + globalThis.fetch = originalFetch + } + }) +}) diff --git a/packages/server/src/workspaces/instance-client.ts b/packages/server/src/workspaces/instance-client.ts index 59607c921..9750090a4 100644 --- a/packages/server/src/workspaces/instance-client.ts +++ b/packages/server/src/workspaces/instance-client.ts @@ -45,11 +45,14 @@ export function createInstanceClient( return createOpencodeClient({ baseUrl: `http://${INSTANCE_HOST}:${port}/`, headers, - fetch: (url, init) => - fetch(url, { - ...(init as RequestInit), - signal: (init as RequestInit)?.signal ?? AbortSignal.timeout(timeoutMs), - }), + fetch: (url, init) => fetch(url, { + ...(init as RequestInit), + signal: AbortSignal.any([ + ...(url instanceof Request ? [url.signal] : []), + ...((init as RequestInit | undefined)?.signal ? [(init as RequestInit).signal!] : []), + AbortSignal.timeout(timeoutMs), + ]), + }), ...(workspace?.path ? { directory: workspace.path } : {}), }) } diff --git a/packages/server/src/workspaces/managed-worktree.test.ts b/packages/server/src/workspaces/managed-worktree.test.ts new file mode 100644 index 000000000..dae74bb21 --- /dev/null +++ b/packages/server/src/workspaces/managed-worktree.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict" +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import { test } from "node:test" + +import { isManagedWorktree } from "./git-worktrees" + +test("managed worktrees require a canonical direct child with Git metadata", async () => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-managed-worktree-")) + const repoRoot = path.join(temp, "repo") + const managed = path.join(repoRoot, ".codenomad", "worktrees", "review") + const metadata = path.join(repoRoot, ".git", "worktrees", "review") + const external = path.join(repoRoot, "..", "external") + try { + mkdirSync(managed, { recursive: true }) + mkdirSync(metadata, { recursive: true }) + mkdirSync(external, { recursive: true }) + writeFileSync(path.join(managed, ".git"), `gitdir: ${metadata}\n`) + + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory: managed, kind: "worktree" }, + }), true) + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory: external, kind: "worktree" }, + }), false) + rmSync(path.join(managed, ".git")) + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory: managed, kind: "worktree" }, + }), false) + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) + +test("managed worktree root cannot redirect outside the repository", async () => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-managed-root-")) + const repoRoot = path.join(temp, "repo") + const externalRoot = path.join(temp, "external-worktrees") + const directory = path.join(externalRoot, "review") + const metadata = path.join(repoRoot, ".git", "worktrees", "review") + try { + mkdirSync(path.join(repoRoot, ".codenomad"), { recursive: true }) + mkdirSync(directory, { recursive: true }) + mkdirSync(metadata, { recursive: true }) + writeFileSync(path.join(directory, ".git"), `gitdir: ${metadata}\n`) + symlinkSync(externalRoot, path.join(repoRoot, ".codenomad", "worktrees"), "junction") + + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory, kind: "worktree" }, + }), false) + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) diff --git a/packages/server/src/workspaces/manager.test.ts b/packages/server/src/workspaces/manager.test.ts index 7b9a66e4d..d5897fa6d 100644 --- a/packages/server/src/workspaces/manager.test.ts +++ b/packages/server/src/workspaces/manager.test.ts @@ -10,6 +10,7 @@ import { } from "./runtime" import { WorkspaceCleanupTimeoutError, + WorkspaceDeletionBlockedError, WorkspaceLaunchCancelledError, WorkspaceLaunchTimeoutError, WorkspaceManager, @@ -33,8 +34,10 @@ class ControlledRuntime { stopCalls = 0 failStops = 0 onExit?: (info: ProcessExitInfo) => void + launchEnvironment?: Record launch: WorkspaceRuntime["launch"] = (options) => { + this.launchEnvironment = options.environment this.active.add(options.workspaceId) this.onExit = options.onExit this.launchCalled.resolve(options.workspaceId) @@ -102,6 +105,115 @@ async function createReady(harness: ReturnType) { } describe("workspace manager lifecycle", () => { + it("guards restore cancellation before aborting or deleting its workspace", async () => { + const harness = createHarness() + const creation = harness.manager.create(process.cwd(), undefined, { requestId: "restore-request" }) + const workspaceId = await harness.runtime.launchCalled.promise + harness.runtime.resolveLaunch() + harness.readiness.resolve(undefined) + await creation + let guardedWorkspaceId: string | undefined + harness.manager.setDeletionGuard(async (workspace) => { + guardedWorkspaceId = workspace.id + throw new WorkspaceDeletionBlockedError(workspace.id) + }) + + await assert.rejects(harness.manager.cancelCreationRequest("restore-request"), WorkspaceDeletionBlockedError) + assert.equal(guardedWorkspaceId, workspaceId) + assert.equal(harness.manager.get(workspaceId)?.status, "ready") + assert.equal(harness.runtime.stopCalls, 0) + + harness.manager.setDeletionGuard(async (_workspace, operation) => operation()) + await harness.manager.cancelCreationRequest("restore-request") + assert.equal(harness.manager.get(workspaceId), undefined) + }) + + it("does not reuse a ready workspace while guarded deletion is pending", async () => { + const harness = createHarness() + const workspaceId = await createReady(harness) + const guardEntered = deferred() + const finishGuard = deferred() + harness.manager.setDeletionGuard(async (workspace) => { + guardEntered.resolve() + await finishGuard.promise + throw new WorkspaceDeletionBlockedError(workspace.id) + }) + + const deletion = harness.manager.delete(workspaceId) + await guardEntered.promise + const replacement = await harness.manager.create(process.cwd()) + + assert.equal(replacement.created, true) + assert.notEqual(replacement.workspace.id, workspaceId) + finishGuard.resolve() + await assert.rejects(deletion, WorkspaceDeletionBlockedError) + assert.equal(harness.manager.get(workspaceId)?.status, "ready") + }) + + it("does not reuse a pending workspace while its guarded deletion is pending", async () => { + const harness = createHarness() + const firstCreation = harness.manager.create(process.cwd()) + const workspaceId = await harness.runtime.launchCalled.promise + const guardEntered = deferred() + const finishGuard = deferred() + harness.manager.setDeletionGuard(async (_workspace, operation) => { + guardEntered.resolve() + await finishGuard.promise + return operation() + }) + + const deletion = harness.manager.delete(workspaceId) + await guardEntered.promise + const replacement = harness.manager.create(process.cwd()) + while ((harness.manager as any).workspaces.size < 2) { + await new Promise((resolve) => setImmediate(resolve)) + } + + assert.equal((harness.manager as any).workspaces.size, 2) + const pending = [...(harness.manager as any).pendingWorkspaceCreations.values()] as Array<{ id: string }> + assert.equal(pending.length, 1) + assert.notEqual(pending[0]?.id, workspaceId) + finishGuard.resolve() + await assert.rejects(firstCreation, WorkspaceLaunchCancelledError) + await deletion + await assert.rejects(replacement, WorkspaceLaunchCancelledError) + }) + + for (const status of ["stopped", "error"] as const) { + it(`does not reuse a ${status} lineage record`, async () => { + const harness = createHarness() + const first = await (async () => { + const creation = harness.manager.create(process.cwd(), undefined, { lineageId: "terminal-lineage" }) + harness.runtime.resolveLaunch() + harness.readiness.resolve(undefined) + return creation + })() + ;(harness.manager.get(first.workspace.id) as { status: string }).status = status + + const second = await harness.manager.create(process.cwd(), undefined, { lineageId: "terminal-lineage" }) + + assert.equal(second.created, true) + assert.notEqual(second.workspace.id, first.workspace.id) + }) + } + + it("uses a distinct generated capability for plugin callbacks", async () => { + const harness = createHarness() + ;(harness.manager as any).options.settings = { + getOwner: () => ({ environmentVariables: { OPENCODE_SERVER_PASSWORD: "shared-opencode-secret" } }), + } + const workspaceId = await createReady(harness) + const callbackToken = harness.runtime.launchEnvironment?.CODENOMAD_CALLBACK_TOKEN + + assert.ok(callbackToken) + assert.notEqual(callbackToken, "shared-opencode-secret") + assert.equal(harness.manager.getPluginCallbackAuthorizationHeader(workspaceId), `Bearer ${callbackToken}`) + assert.equal( + harness.manager.getInstanceAuthorizationHeader(workspaceId), + `Basic ${Buffer.from("codenomad:shared-opencode-secret").toString("base64")}`, + ) + }) + for (const boundary of ["launch", "readiness", "shutdown"] as const) { it(`cancels and cleans a workspace during ${boundary}`, async () => { const harness = createHarness() diff --git a/packages/server/src/workspaces/manager.ts b/packages/server/src/workspaces/manager.ts index 547bccd8f..ffeed3e52 100644 --- a/packages/server/src/workspaces/manager.ts +++ b/packages/server/src/workspaces/manager.ts @@ -18,8 +18,11 @@ import { resolveExistingOpencodeConfigContent, } from "../opencode-plugin.js" import { + CODENOMAD_CALLBACK_TOKEN_ENV, OPENCODE_SERVER_BASE_URL_ENV, + buildCodeNomadCallbackAuthorizationHeader, buildOpencodeBasicAuthHeader, + generateCodeNomadCallbackToken, OPENCODE_SERVER_PASSWORD_ENV, OPENCODE_SERVER_USERNAME_ENV, resolveOpencodeServerAuth, @@ -69,6 +72,11 @@ interface WorkspaceManagerOptions { clearTimeout?: (timer: ManagerTimeout) => void } +export type WorkspaceDeletionGuard = ( + workspace: WorkspaceDescriptor, + operation: () => Promise, +) => Promise + interface WorkspaceRecord extends WorkspaceDescriptor { identityKey: string ownership: WorkspaceCreationOwnership @@ -105,6 +113,13 @@ export class WorkspaceCleanupTimeoutError extends Error { this.name = "WorkspaceCleanupTimeoutError" } } +export class WorkspaceDeletionBlockedError extends Error { + readonly code = "WORKSPACE_OWNED_BY_WORKFLOW" + constructor(workspaceId: string) { + super(`Workspace ${workspaceId} is owned by an active workflow`) + this.name = "WorkspaceDeletionBlockedError" + } +} export class WorkspaceShutdownError extends AggregateError { readonly code = "WORKSPACE_SHUTDOWN_FAILED" readonly retryable = true @@ -121,6 +136,7 @@ export interface WorkspaceCreateOptions { binaryPath?: string requestId?: string forceNew?: boolean + lineageId?: string } type CreationRequestState = "active" | "cancelled" | "released" type WorkspaceCreationOwnership = Map @@ -139,6 +155,8 @@ export class WorkspaceManager { private readonly runtime: Pick private readonly codeNomadPluginUrl: string private readonly opencodeAuth = new Map() + private readonly pluginCallbackAuth = new Map() + private deletionGuard?: WorkspaceDeletionGuard constructor(private readonly options: WorkspaceManagerOptions) { this.runtime = options.runtime ?? new WorkspaceRuntime(this.options.eventBus, this.options.logger) @@ -163,6 +181,14 @@ export class WorkspaceManager { return this.workspaces.get(id)?.[WORKSPACE_STATE].published ? this.opencodeAuth.get(id)?.authorization : undefined } + getPluginCallbackAuthorizationHeader(id: string): string | undefined { + return this.workspaces.get(id)?.[WORKSPACE_STATE].published ? this.pluginCallbackAuth.get(id) : undefined + } + + setDeletionGuard(guard: WorkspaceDeletionGuard): void { + this.deletionGuard = guard + } + findReadyInstanceIdByBinary(binaryPath: string): string | undefined { const resolvedPath = this.resolveBinaryPath(binaryPath) return this.list().find((workspace) => { @@ -179,6 +205,7 @@ export class WorkspaceManager { if ( state.published && !state.abortController.signal.aborted + && !state.deletePromise && (includeRestoreOwned || !record.requestId) && record.status === "ready" && record.identityKey === identityKey @@ -258,7 +285,26 @@ export class WorkspaceManager { if (this.shuttingDown) { throw new Error("Workspace manager is shutting down") } - if (options.forceNew) { + if (options.lineageId) { + const lineageRecord = Array.from(this.workspaces.values()).find((record) => + record.lineageId === options.lineageId + && (record.status === "starting" || record.status === "ready") + && !record[WORKSPACE_STATE].deletePromise + && !record[WORKSPACE_STATE].abortController.signal.aborted) + if (lineageRecord) { + if (lineageRecord.identityKey !== identityKey) { + throw new Error("Workspace lineage belongs to a different workspace") + } + const owner = options.requestId ?? ORDINARY_CREATION_OWNER + if (!lineageRecord.ownership.has(owner)) lineageRecord.ownership.set(owner, "active") + this.syncOwnership(lineageRecord) + const workspace = lineageRecord[WORKSPACE_STATE].creation + ? (await lineageRecord[WORKSPACE_STATE].creation).workspace + : lineageRecord + return this.finishCreation({ workspace, created: false }, options.requestId, lineageRecord.ownership) + } + } + if (options.forceNew || options.lineageId) { const ownership = this.createOwnership(options.requestId) const record = this.reserveWorkspace(workspacePath, identityKey, name, options, ownership, launchDeadlineAt) const result = await this.startCreation(record, options, launchDeadlineAt, launchTimeoutMs) @@ -276,7 +322,12 @@ export class WorkspaceManager { return { workspace: existing, created: false } } const pending = this.pendingWorkspaceCreations.get(identityKey) - if (pending) { + if ( + pending + && !pending[WORKSPACE_STATE].deletePromise + && !pending[WORKSPACE_STATE].abortController.signal.aborted + && (pending.status === "starting" || pending.status === "ready") + ) { const state = pending[WORKSPACE_STATE] const owner = options.requestId ?? ORDINARY_CREATION_OWNER if (!pending.ownership.has(owner)) pending.ownership.set(owner, "active") @@ -319,6 +370,7 @@ export class WorkspaceManager { const record = { id, + lineageId: options.lineageId ?? randomUUID(), requestId: options.requestId, path: workspacePath, name, @@ -410,6 +462,8 @@ export class WorkspaceManager { throw new Error("Failed to build OpenCode auth header") } this.opencodeAuth.set(id, { username: opencodeUsername, password: opencodePassword, authorization }) + const callbackToken = generateCodeNomadCallbackToken() + this.pluginCallbackAuth.set(id, buildCodeNomadCallbackAuthorizationHeader(callbackToken)) const environment = { ...userEnvironment, @@ -417,6 +471,7 @@ export class WorkspaceManager { OPENCODE_EXPERIMENTAL_WORKSPACES: "true", CODENOMAD_INSTANCE_ID: id, CODENOMAD_BASE_URL: serverBaseUrl, + [CODENOMAD_CALLBACK_TOKEN_ENV]: callbackToken, ...(this.options.nodeExtraCaCertsPath ? { NODE_EXTRA_CA_CERTS: this.options.nodeExtraCaCertsPath } : {}), [OPENCODE_SERVER_BASE_URL_ENV]: `${normalizedServerBaseUrl}${proxyPath}`, [OPENCODE_SERVER_USERNAME_ENV]: opencodeUsername, @@ -487,19 +542,24 @@ export class WorkspaceManager { const record = this.workspaces.get(id) if (!record) return Promise.resolve(undefined) const state = record[WORKSPACE_STATE] - if (!state.abortController.signal.aborted) { - state.abortController.abort(new WorkspaceLaunchCancelledError(id)) - } - const pending = this.pendingWorkspaceCreations.get(record.identityKey) - if (pending === record) { - this.pendingWorkspaceCreations.delete(record.identityKey) - } if (!state.deletePromise) { - let deletePromise!: Promise - deletePromise = this.cleanupDeletedWorkspace(id, record).catch((error) => { - if (state.deletePromise === deletePromise) state.deletePromise = undefined - throw error - }) + const cleanup = () => { + if (!state.abortController.signal.aborted) { + state.abortController.abort(new WorkspaceLaunchCancelledError(id)) + } + if (this.pendingWorkspaceCreations.get(record.identityKey) === record) { + this.pendingWorkspaceCreations.delete(record.identityKey) + } + return this.cleanupDeletedWorkspace(id, record) + } + const deletePromise = Promise.resolve() + .then(() => this.shuttingDown || !this.deletionGuard + ? cleanup() + : this.deletionGuard(record, cleanup)) + .catch((error) => { + if (state.deletePromise === deletePromise) state.deletePromise = undefined + throw error + }) state.deletePromise = deletePromise } return state.deletePromise @@ -637,6 +697,7 @@ export class WorkspaceManager { if (this.workspaces.get(id) !== record) return this.workspaces.delete(id) this.opencodeAuth.delete(id) + this.pluginCallbackAuth.delete(id) clearWorkspaceSearchCache(record.path) if (publishStopped) this.publishStopped(record, "deleted") } @@ -857,6 +918,7 @@ export class WorkspaceManager { const workspace = record this.opencodeAuth.delete(workspaceId) + this.pluginCallbackAuth.delete(workspaceId) this.options.logger.info({ workspaceId, ...info }, "Workspace process exited") diff --git a/packages/server/src/workspaces/opencode-auth.test.ts b/packages/server/src/workspaces/opencode-auth.test.ts index e4a13a4d5..ea766592f 100644 --- a/packages/server/src/workspaces/opencode-auth.test.ts +++ b/packages/server/src/workspaces/opencode-auth.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" -import { resolveOpencodeServerAuth } from "./opencode-auth" +import { buildCodeNomadCallbackAuthorizationHeader, generateCodeNomadCallbackToken, resolveOpencodeServerAuth } from "./opencode-auth" describe("resolveOpencodeServerAuth", () => { it("uses configured OpenCode auth from workspace environment", () => { @@ -39,3 +39,9 @@ describe("resolveOpencodeServerAuth", () => { assert.deepEqual(auth, { username: "codenomad", password: "generated" }) }) }) + +it("builds a bearer callback capability", () => { + const token = generateCodeNomadCallbackToken() + assert.match(token, /^[A-Za-z0-9_-]{43}$/) + assert.equal(buildCodeNomadCallbackAuthorizationHeader(token), `Bearer ${token}`) +}) diff --git a/packages/server/src/workspaces/opencode-auth.ts b/packages/server/src/workspaces/opencode-auth.ts index 55daeed7b..c2ad332bc 100644 --- a/packages/server/src/workspaces/opencode-auth.ts +++ b/packages/server/src/workspaces/opencode-auth.ts @@ -3,6 +3,7 @@ import crypto from "node:crypto" export const OPENCODE_SERVER_USERNAME_ENV = "OPENCODE_SERVER_USERNAME" as const export const OPENCODE_SERVER_PASSWORD_ENV = "OPENCODE_SERVER_PASSWORD" as const export const OPENCODE_SERVER_BASE_URL_ENV = "OPENCODE_SERVER_BASE_URL" as const +export const CODENOMAD_CALLBACK_TOKEN_ENV = "CODENOMAD_CALLBACK_TOKEN" as const export const DEFAULT_OPENCODE_USERNAME = "codenomad" as const @@ -10,6 +11,10 @@ export function generateOpencodeServerPassword(): string { return crypto.randomBytes(32).toString("base64url") } +export function generateCodeNomadCallbackToken(): string { + return crypto.randomBytes(32).toString("base64url") +} + function readConfiguredValue(key: string, ...sources: Array | undefined>): string | undefined { for (const source of sources) { const value = source?.[key] @@ -47,3 +52,7 @@ export function buildOpencodeBasicAuthHeader(params: { username?: string; passwo const token = Buffer.from(`${username}:${password}`, "utf8").toString("base64") return `Basic ${token}` } + +export function buildCodeNomadCallbackAuthorizationHeader(token: string): string { + return `Bearer ${token}` +} diff --git a/packages/ui/src/components/agent-selector.tsx b/packages/ui/src/components/agent-selector.tsx index be3616d35..5ede2af68 100644 --- a/packages/ui/src/components/agent-selector.tsx +++ b/packages/ui/src/components/agent-selector.tsx @@ -13,6 +13,8 @@ interface AgentSelectorProps { sessionId: string currentAgent: string onAgentChange: (agent: string) => Promise + allowChildAgents?: boolean + square?: boolean } export default function AgentSelector(props: AgentSelectorProps) { @@ -25,7 +27,7 @@ export default function AgentSelector(props: AgentSelectorProps) { }) const isChildSession = createMemo(() => { - return session()?.parentId !== null && session()?.parentId !== undefined + return props.allowChildAgents || (session()?.parentId !== null && session()?.parentId !== undefined) }) const availableAgents = createMemo(() => { @@ -105,7 +107,7 @@ export default function AgentSelector(props: AgentSelectorProps) { - + diff --git a/packages/ui/src/components/instance/shell/right-panel/RightPanel.tsx b/packages/ui/src/components/instance/shell/right-panel/RightPanel.tsx index 64f195471..04cc23346 100644 --- a/packages/ui/src/components/instance/shell/right-panel/RightPanel.tsx +++ b/packages/ui/src/components/instance/shell/right-panel/RightPanel.tsx @@ -5,6 +5,7 @@ import { createEffect, createMemo, createSignal, + createUniqueId, lazy, onCleanup, type Accessor, @@ -84,6 +85,7 @@ import { CORE_STATUS_SECTION_ITEMS } from "./tabs/status-sections" const LazyGitChangesTab = lazy(() => import("./tabs/GitChangesTab")) const LazyFilesTab = lazy(() => import("./tabs/FilesTab")) +const LazyWorkflowsTab = lazy(() => import("./tabs/WorkflowsTab")) const LazyStatusTab = lazy(() => import("./tabs/StatusTab")) function RightPanelTabFallback() { @@ -95,7 +97,11 @@ interface SortableRightPanelTabProps { active: boolean label: string dragTitle: string + id: string + panelId: string + tabIndex: number onSelect: () => void + onKeyDown: (event: KeyboardEvent) => void } const SortableRightPanelTab: Component = (props) => { @@ -105,9 +111,13 @@ const SortableRightPanelTab: Component = (props) =>