diff --git a/docs/superpowers/plans/2026-07-13-automatic-oracle-policy.md b/docs/superpowers/plans/2026-07-13-automatic-oracle-policy.md new file mode 100644 index 000000000..dec8bb648 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-automatic-oracle-policy.md @@ -0,0 +1,35 @@ +# Automatic Oracle Consultation Policy Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Automatically and audibly recommend or require the read-only Oracle for tasks where independent reasoning is likely to improve outcomes. + +**Architecture:** A pure versioned policy evaluates profile, task type, prompt signals, and prior failures. The intelligent router records the decision, and the chat handler injects explicit next-run guidance; a checked-in eval matrix prevents trigger drift. + +**Tech Stack:** TypeScript, Vitest, intelligent router, Agent system-prompt additions. + +## Global Constraints + +- No hidden tool execution; the agent sees and follows an explicit policy directive. +- Oracle remains read-only and uses the profile’s complementary model. +- Low-risk profiles avoid mandatory Oracle cost. +- Policy behavior must be covered by named eval cases before integration. + +--- + +### Task 1: Versioned consultation policy + +**Files:** Create `src/agent/oracle-consultation-policy.ts`; test `test/agent/oracle-consultation-policy.test.ts`. + +- [x] Write a failing eval matrix for low, medium, high, ultra, architecture/migration, ambiguity, cross-cutting work, and repeated failures. +- [x] Implement minimal deterministic scoring and prompt directive formatting. +- [x] Verify all eval cases pass. + +### Task 2: Router and runtime integration + +**Files:** Modify intelligent-router types/normalization/service/recorder and `src/server/handlers/chat.ts`; update focused tests. + +- [x] Add failing tests for decision recording and next-run prompt injection. +- [x] Thread task summary and prior failure count through routing. +- [x] Queue policy guidance only for recommended/required modes. +- [ ] Run targeted tests, lint, affected tests, commit, open PR after PR 2 merges, address feedback, merge, and verify deployment/mirror workflows. diff --git a/src/agent/oracle-consultation-policy.ts b/src/agent/oracle-consultation-policy.ts new file mode 100644 index 000000000..4122b526d --- /dev/null +++ b/src/agent/oracle-consultation-policy.ts @@ -0,0 +1,110 @@ +import type { AgentProfileLevel } from "./profiles.js"; + +export const ORACLE_CONSULTATION_POLICY_VERSION = + "evalops.maestro.oracle-consultation.v1"; + +export type OracleConsultationMode = "available" | "recommended" | "required"; + +export interface OracleConsultationPolicyInput { + profileLevel: AgentProfileLevel; + taskType: string; + taskSummary?: string; + priorFailures?: number; +} + +export interface OracleConsultationDecision { + policyVersion: typeof ORACLE_CONSULTATION_POLICY_VERSION; + evalSuite: "oracle-consultation-policy-v1"; + mode: OracleConsultationMode; + reasons: string[]; +} + +const CONSULTATION_TASK_TYPES = new Set([ + "architecture", + "code_review", + "discovery", + "incident_response", + "migration", + "planning", + "security_review", +]); + +const UNCERTAINTY_PATTERN = + /\b(?:ambiguous|unclear|uncertain|trade-?offs?|cross[- ]cutting|multiple approaches|data loss|irreversible|root cause unknown)\b/i; + +const CONSULTATION_PROMPT_PATTERN = + /\b(?:architect(?:ure|ural|ing)?|migrat(?:e|es|ed|ing|ion)|security|secure|threat model(?:ing)?|auth(?:entication|orization)?|schema (?:change|migration)|backfill)\b/i; + +export function recommendOracleConsultation( + input: OracleConsultationPolicyInput, +): OracleConsultationDecision { + const reasons: string[] = []; + const priorFailures = Math.max(0, Math.floor(input.priorFailures ?? 0)); + let mode: OracleConsultationMode = "available"; + + if (input.profileLevel === "ultra") { + mode = "required"; + reasons.push("ultra_profile"); + } else if (input.profileLevel === "high") { + mode = "recommended"; + reasons.push("high_profile"); + } + + if (CONSULTATION_TASK_TYPES.has(input.taskType.trim().toLowerCase())) { + if (mode === "available") mode = "recommended"; + reasons.push("consultation_task_type"); + } + + if (input.taskSummary && UNCERTAINTY_PATTERN.test(input.taskSummary)) { + if (mode === "available") mode = "recommended"; + reasons.push("uncertainty_signal"); + } + + if ( + input.taskSummary && + CONSULTATION_PROMPT_PATTERN.test(input.taskSummary) + ) { + if (mode === "available") mode = "recommended"; + reasons.push("consultation_prompt_signal"); + } + + if (priorFailures >= 2) { + mode = input.profileLevel === "low" ? "recommended" : "required"; + reasons.push("repeated_failures"); + } + + if (reasons.length === 0) reasons.push("oracle_available_on_demand"); + return { + policyVersion: ORACLE_CONSULTATION_POLICY_VERSION, + evalSuite: "oracle-consultation-policy-v1", + mode, + reasons, + }; +} + +export function formatOracleConsultationDirective( + decision: OracleConsultationDecision, +): string { + const instruction = + decision.mode === "required" + ? "You MUST consult the read-only Oracle once before committing to the plan or final answer. Incorporate or explicitly rebut its advice." + : decision.mode === "recommended" + ? "Consult the read-only Oracle once before committing to the plan or final answer unless the task has become clearly bounded; if you skip it, state the concrete reason." + : "The read-only Oracle is available on demand."; + return [ + `Oracle consultation policy (${decision.policyVersion})`, + instruction, + `Triggers: ${decision.reasons.join(", ")}.`, + ].join("\n"); +} + +export function applyOracleConsultationDirective( + agent: { queueNextRunSystemPromptAddition(text: string): void }, + decision: OracleConsultationDecision | undefined, +): boolean { + if (!decision || decision.mode === "available") return false; + agent.queueNextRunSystemPromptAddition( + formatOracleConsultationDirective(decision), + ); + return true; +} diff --git a/src/server/handlers/chat-ws.ts b/src/server/handlers/chat-ws.ts index 28919bc8a..42392f7d6 100644 --- a/src/server/handlers/chat-ws.ts +++ b/src/server/handlers/chat-ws.ts @@ -4,9 +4,11 @@ * Mirrors the SSE chat flow but streams AgentEvent payloads over WebSocket. */ +import { randomBytes, randomUUID } from "node:crypto"; import type { IncomingMessage } from "node:http"; import type { ComposerChatRequest, ComposerMessage } from "@evalops/contracts"; import type { RawData, WebSocket } from "ws"; +import { applyOracleConsultationDirective } from "../../agent/oracle-consultation-policy.js"; import { isAssistantMessage } from "../../agent/type-guards.js"; import type { Attachment as AgentAttachment, @@ -22,7 +24,13 @@ import { } from "../../memory/auto-consolidation.js"; import { createAutomaticMemoryExtractionCoordinator } from "../../memory/auto-extraction.js"; import type { RegisteredModel } from "../../models/registry.js"; +import { + recordIntelligentRouterChatMetric, + selectIntelligentRouterModel, +} from "../../services/intelligent-router/recorder.js"; import { recordAssistantUsageMetric } from "../../services/usage-analytics/recorder.js"; +import { evaluateModelPolicy } from "../../services/workspace-config/policy.js"; +import type { WorkspaceConfigRequestContext } from "../../services/workspace-config/types.js"; import { toSessionModelMetadata } from "../../session/manager.js"; import { createRuntimeSessionSummaryUpdater } from "../../session/runtime-summary-updater.js"; import { createLogger } from "../../utils/logger.js"; @@ -37,6 +45,13 @@ import { getAuthSubject } from "../authz.js"; import { getAgentCircuitBreaker } from "../circuit-breaker.js"; import { clientToolService } from "../client-tools-service.js"; import { isHostedSessionManager } from "../hosted-session-manager.js"; +import { resolveModelInputForRouting } from "../model-selection.js"; +import { + type RequestContext, + getWorkspaceConfigContext, + parseTraceParent, + requestContextStorage, +} from "../request-context.js"; import { serverRequestManager } from "../server-request-manager.js"; import { getRequestHeader } from "../server-utils.js"; import { startSessionWithPolicy } from "../session-initialization.js"; @@ -219,12 +234,15 @@ export function handleChatWebSocket( ws: WebSocket, req: IncomingMessage, context: WebServerContext, + workspaceConfig?: WorkspaceConfigRequestContext, ) { const { createAgent, createBackgroundAgent, getRegisteredModel, defaultApprovalMode, + defaultProvider, + defaultModelId, acquireSse, releaseSse, } = context; @@ -245,6 +263,17 @@ export function handleChatWebSocket( ); const slimFromQuery = parseBoolean(url.searchParams.get("slim")); const clientHeaderFromQuery = url.searchParams.get("client")?.trim(); + const websocketRequestContext: RequestContext | undefined = workspaceConfig + ? { + requestId: randomUUID(), + traceId: parseTraceParent(req.headers.traceparent).traceId, + spanId: randomBytes(8).toString("hex"), + startTime: performance.now(), + method: req.method || "GET", + url: url.pathname, + workspaceConfig, + } + : undefined; const sendErrorAndClose = (message: string) => { try { @@ -271,7 +300,7 @@ export function handleChatWebSocket( return validatePayload(parsed, ChatRequestSchema); }; - ws.on("message", async (data) => { + const handleMessage = async (data: RawData) => { if (requestHandled) { try { const size = getRawDataSize(data); @@ -450,7 +479,55 @@ export function handleChatWebSocket( } } - const registeredModel = await getRegisteredModel(chatReq.model); + const routingSelection = selectIntelligentRouterModel({ + req, + requestedModel: resolveModelInputForRouting( + chatReq.model, + defaultProvider, + defaultModelId, + ), + body: chatReq, + }); + let registeredModel: RegisteredModel | undefined; + let lastModelError: unknown; + let selectedModelError: unknown; + let selectedModelPolicyViolation: ReturnType< + typeof evaluateModelPolicy + > | null = null; + for (const [ + index, + modelInput, + ] of routingSelection.modelInputs.entries()) { + try { + const candidateModel = await getRegisteredModel(modelInput); + const violation = evaluateModelPolicy( + workspaceConfig?.config ?? getWorkspaceConfigContext()?.config, + { + provider: candidateModel.provider, + modelId: candidateModel.id, + }, + ); + if (violation) { + if (index === 0) selectedModelPolicyViolation = violation; + continue; + } + registeredModel = candidateModel; + break; + } catch (error) { + lastModelError = error; + if (index === 0) selectedModelError = error; + } + } + if (!registeredModel && selectedModelPolicyViolation) { + sendErrorAndClose(selectedModelPolicyViolation.message); + if (sseLease && releaseSse) { + releaseSse(sseLease); + sseLease = null; + } + return; + } + if (!registeredModel) throw selectedModelError ?? lastModelError; + const routeStartedAt = Date.now(); const headerApproval = (() => { const headerMode = normalizeApprovalMode( @@ -529,6 +606,10 @@ export function handleChatWebSocket( : {}), }, ); + applyOracleConsultationDirective( + agent, + routingSelection.decision.oracleConsultation, + ); const historyMessages = incomingMessages.slice(0, -1); const hydratedHistory = convertComposerMessagesToApp( @@ -844,6 +925,13 @@ export function handleChatWebSocket( sessionId: sessionManager.getSessionId(), subject, }); + recordIntelligentRouterChatMetric({ + taskType: routingSelection.taskType, + provider: registeredModel.provider, + model: registeredModel.id, + startedAt: routeStartedAt, + message: event.message, + }); automaticMemoryExtraction.schedule(sessionManager.getSessionFile()); } sessionManager.updateSnapshot( @@ -962,5 +1050,14 @@ export function handleChatWebSocket( sseLease = null; } } + }; + + ws.on("message", (data) => { + if (websocketRequestContext) { + return requestContextStorage.run(websocketRequestContext, () => + handleMessage(data), + ); + } + return handleMessage(data); }); } diff --git a/src/server/handlers/chat.ts b/src/server/handlers/chat.ts index 18818fecb..7c455bbc9 100644 --- a/src/server/handlers/chat.ts +++ b/src/server/handlers/chat.ts @@ -23,6 +23,7 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import type { ComposerChatRequest, ComposerMessage } from "@evalops/contracts"; +import { applyOracleConsultationDirective } from "../../agent/oracle-consultation-policy.js"; import { isAssistantMessage } from "../../agent/type-guards.js"; import type { Attachment as AgentAttachment, @@ -63,6 +64,7 @@ import { getAuthSubject } from "../authz.js"; import { getAgentCircuitBreaker } from "../circuit-breaker.js"; import { clientToolService } from "../client-tools-service.js"; import { isHostedSessionManager } from "../hosted-session-manager.js"; +import { resolveModelInputForRouting } from "../model-selection.js"; import { getWorkspaceConfigContext } from "../request-context.js"; import { serverRequestManager } from "../server-request-manager.js"; import { @@ -131,6 +133,8 @@ export async function handleChat( createBackgroundAgent, getRegisteredModel, defaultApprovalMode, + defaultProvider, + defaultModelId, acquireSse, releaseSse, corsHeaders: cors, @@ -319,7 +323,11 @@ export async function handleChat( // it can select a better-scoring model and expose explicit fallbacks. const routingSelection = selectIntelligentRouterModel({ req, - requestedModel: chatReq.model, + requestedModel: resolveModelInputForRouting( + chatReq.model, + defaultProvider, + defaultModelId, + ), body: chatReq, }); let registeredModel: RegisteredModel | undefined; @@ -434,6 +442,10 @@ export async function handleChat( : {}), }, ); + applyOracleConsultationDirective( + agent, + routingSelection.decision.oracleConsultation, + ); // Hydrate conversation history (all messages except the current user input) const historyMessages = incomingMessages.slice(0, -1); diff --git a/src/server/model-selection.ts b/src/server/model-selection.ts index c61b74054..af57d9cba 100644 --- a/src/server/model-selection.ts +++ b/src/server/model-selection.ts @@ -130,6 +130,19 @@ export function determineModelSelection( }; } +export function resolveModelInputForRouting( + modelInput: string | null | undefined, + defaultProvider: string, + defaultModelId: string, +): string { + const selection = determineModelSelection( + modelInput, + defaultProvider, + defaultModelId, + ); + return `${selection.provider}/${selection.modelId}`; +} + export function getRegisteredModelOrThrow( selection: ModelSelection, ): RegisteredModel { diff --git a/src/services/intelligent-router/normalize.ts b/src/services/intelligent-router/normalize.ts index 22a534006..93b3c7d26 100644 --- a/src/services/intelligent-router/normalize.ts +++ b/src/services/intelligent-router/normalize.ts @@ -182,6 +182,18 @@ export function normalizeRoutingRequest( const availableModels = normalizeAvailableModels( requestInput.availableModels, ); + const taskSummary = + cleanOptionalString(requestInput.taskSummary) ?? + cleanOptionalString(requestInput.task_summary); + const priorFailures = Math.max( + 0, + Math.floor( + parseFiniteNumber( + requestInput.priorFailures ?? requestInput.prior_failures, + "priorFailures", + ) ?? 0, + ), + ); return { taskType, profileLevel, @@ -189,7 +201,9 @@ export function normalizeRoutingRequest( unavailableModels, availableModels: availableModels.length > 0 ? availableModels : defaultAvailableModels, + priorFailures, ...(modelHint ? { modelHint } : {}), + ...(taskSummary ? { taskSummary } : {}), }; } diff --git a/src/services/intelligent-router/recorder.ts b/src/services/intelligent-router/recorder.ts index 62304dc8e..4603a9b82 100644 --- a/src/services/intelligent-router/recorder.ts +++ b/src/services/intelligent-router/recorder.ts @@ -79,6 +79,62 @@ export function resolveIntelligentRouterStrategy( : undefined; } +export function resolveIntelligentRouterTaskSummary( + body: unknown, +): string | undefined { + if (!body || typeof body !== "object") return undefined; + if ("taskSummary" in body) { + const summary = (body as { taskSummary?: unknown }).taskSummary; + if (typeof summary === "string" && summary.trim()) return summary.trim(); + } + if ( + !("messages" in body) || + !Array.isArray((body as { messages?: unknown }).messages) + ) { + return undefined; + } + const messages = (body as { messages: unknown[] }).messages; + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (!message || typeof message !== "object" || !("content" in message)) + continue; + if ("role" in message && message.role !== "user") continue; + const content = (message as { content?: unknown }).content; + if (typeof content === "string" && content.trim()) return content.trim(); + if (Array.isArray(content)) { + const text = content + .map((part) => + part && + typeof part === "object" && + "text" in part && + typeof part.text === "string" + ? part.text + : "", + ) + .filter(Boolean) + .join("\n") + .trim(); + if (text) return text; + } + } + return undefined; +} + +function resolvePriorFailures(req: IncomingMessage, body: unknown): number { + const header = firstHeader(req, [ + "x-maestro-prior-failures", + "x-composer-prior-failures", + ]); + const bodyValue = + body && typeof body === "object" + ? ((body as { priorFailures?: unknown; prior_failures?: unknown }) + .priorFailures ?? + (body as { prior_failures?: unknown }).prior_failures) + : undefined; + const parsed = Number(header ?? bodyValue ?? 0); + return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0; +} + export function selectIntelligentRouterModel(params: { req: IncomingMessage; requestedModel?: string | null; @@ -100,6 +156,8 @@ export function selectIntelligentRouterModel(params: { : undefined); const decision = getIntelligentRouterService().routeRequest({ taskType, + taskSummary: resolveIntelligentRouterTaskSummary(params.body), + priorFailures: resolvePriorFailures(params.req, params.body), availableModels: registeredRoutingModels(), ...(modelHint ? { modelHint } : {}), ...(strategy ? { strategy } : {}), diff --git a/src/services/intelligent-router/service.ts b/src/services/intelligent-router/service.ts index 7acef6f31..51bfc0223 100644 --- a/src/services/intelligent-router/service.ts +++ b/src/services/intelligent-router/service.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import type { ModelProvider } from "../../agent/modes.js"; +import { recommendOracleConsultation } from "../../agent/oracle-consultation-policy.js"; import { type AgentProfile, resolveAgentProfile, @@ -484,6 +485,12 @@ export class IntelligentRouterService { overrideApplied, reason, createdAt, + oracleConsultation: recommendOracleConsultation({ + profileLevel: selectedProfile.level, + taskType: request.taskType, + taskSummary: request.taskSummary, + priorFailures: request.priorFailures, + }), ...(request.modelHint ? { modelHint: request.modelHint } : {}), }; this.decisions.unshift(decision); diff --git a/src/services/intelligent-router/types.ts b/src/services/intelligent-router/types.ts index bb21b21b8..6e96bf8b5 100644 --- a/src/services/intelligent-router/types.ts +++ b/src/services/intelligent-router/types.ts @@ -1,3 +1,4 @@ +import type { OracleConsultationDecision } from "../../agent/oracle-consultation-policy.js"; import type { AgentProfile, AgentProfileLevel } from "../../agent/profiles.js"; export const ROUTING_STRATEGIES = [ @@ -74,6 +75,10 @@ export interface RoutingRequestInput { unavailableModels?: string[] | string; unavailable_models?: string[] | string; availableModels?: RoutingModelCandidate[]; + taskSummary?: string; + task_summary?: string; + priorFailures?: number; + prior_failures?: number; } export interface RoutingRequest { @@ -83,6 +88,8 @@ export interface RoutingRequest { strategy: RoutingStrategy; unavailableModels: string[]; availableModels: RoutingModelCandidate[]; + taskSummary?: string; + priorFailures: number; } export interface RoutingScore { @@ -121,6 +128,7 @@ export interface RoutingDecision { overrideApplied: boolean; reason: string; createdAt: string; + oracleConsultation?: OracleConsultationDecision; } export interface RoutingOverrideInput { diff --git a/src/services/workspace-config/middleware.ts b/src/services/workspace-config/middleware.ts index 724a7fa92..4f7799370 100644 --- a/src/services/workspace-config/middleware.ts +++ b/src/services/workspace-config/middleware.ts @@ -9,7 +9,7 @@ import { WorkspaceConfigUnavailableError, getWorkspaceConfigService, } from "./service.js"; -import type { WorkspaceConfig } from "./types.js"; +import type { WorkspaceConfigRequestContext } from "./types.js"; const logger = createLogger("workspace-config:middleware"); @@ -62,6 +62,30 @@ export function resolveWorkspaceConfigId(req: IncomingMessage): string { ); } +export async function loadWorkspaceConfigRequestContext( + req: IncomingMessage, +): Promise { + const workspaceId = resolveWorkspaceConfigId(req); + const service = getWorkspaceConfigService(); + if (!service.isConfigured()) { + return { workspaceId, config: null, source: "unconfigured" }; + } + + try { + const config = await service.getConfig(workspaceId); + return { + workspaceId, + config, + source: config ? "database" : "missing", + }; + } catch (error) { + if (error instanceof WorkspaceConfigUnavailableError) { + return { workspaceId, config: null, source: "unconfigured" }; + } + throw error; + } +} + export function createWorkspaceConfigMiddleware( corsHeaders: Record, ): Middleware { @@ -72,31 +96,10 @@ export function createWorkspaceConfigMiddleware( return; } - const workspaceId = resolveWorkspaceConfigId(req); - const service = getWorkspaceConfigService(); - if (!service.isConfigured()) { - setWorkspaceConfigContext({ - workspaceId, - config: null, - source: "unconfigured", - }); - await next(); - return; - } - - let config: WorkspaceConfig | null; + let workspaceConfig: WorkspaceConfigRequestContext; try { - config = await service.getConfig(workspaceId); + workspaceConfig = await loadWorkspaceConfigRequestContext(req); } catch (error) { - if (error instanceof WorkspaceConfigUnavailableError) { - setWorkspaceConfigContext({ - workspaceId, - config: null, - source: "unconfigured", - }); - await next(); - return; - } if (error instanceof WorkspaceConfigValidationError) { sendJson(res, 400, { error: error.message }, corsHeaders, req); return; @@ -105,7 +108,7 @@ export function createWorkspaceConfigMiddleware( error: sanitizeWithStaticMask( error instanceof Error ? error.message : String(error), ), - workspaceId, + workspaceId: resolveWorkspaceConfigId(req), }); sendJson( res, @@ -117,11 +120,7 @@ export function createWorkspaceConfigMiddleware( return; } - setWorkspaceConfigContext({ - workspaceId, - config, - source: config ? "database" : "missing", - }); + setWorkspaceConfigContext(workspaceConfig); await next(); }; } diff --git a/src/session/agent-lineage-projection.ts b/src/session/agent-lineage-projection.ts index 99a1e10a0..60e91693a 100644 --- a/src/session/agent-lineage-projection.ts +++ b/src/session/agent-lineage-projection.ts @@ -3,6 +3,7 @@ import type { ToolCall, ToolResultMessage, } from "../agent/types.js"; +import { canonicalCodexSubagentTool } from "../codex/subagent-workgraph.js"; import type { SessionTreeEntry } from "./types.js"; export interface AgentLineageOperation { @@ -49,6 +50,11 @@ interface WorkGraphRecord { }>; } +interface WorkGraphFallback { + toolCallId?: string; + operation?: string; +} + export function buildAgentLineageProjection( entries: SessionTreeEntry[], ): AgentLineageProjection { @@ -104,25 +110,38 @@ function workGraphsFromMessage(message: AppMessage): WorkGraphRecord[] { if (message.role === "assistant" && Array.isArray(message.content)) { return message.content .filter(isToolCall) - .map((toolCall) => parseWorkGraph(toolCall.arguments)) + .map((toolCall) => + parseWorkGraph(toolCall.arguments, { + toolCallId: toolCall.id, + operation: canonicalCodexSubagentTool(toolCall.name) ?? toolCall.name, + }), + ) .filter((graph): graph is WorkGraphRecord => Boolean(graph)); } if (message.role === "toolResult") { const result = message as unknown as ToolResultMessage; - const graph = parseWorkGraph(result.details); + const graph = parseWorkGraph(result.details, { + toolCallId: result.toolCallId, + operation: canonicalCodexSubagentTool(result.toolName) ?? result.toolName, + }); return graph ? [graph] : []; } return []; } -function parseWorkGraph(container: unknown): WorkGraphRecord | undefined { +function parseWorkGraph( + container: unknown, + fallback: WorkGraphFallback = {}, +): WorkGraphRecord | undefined { if (!isRecord(container)) return undefined; const graphValue = container.codexWorkGraph ?? container.codex_work_graph; if (!isRecord(graphValue)) return undefined; const toolCallId = stringValue( - graphValue.toolCallId ?? graphValue.tool_call_id, + graphValue.toolCallId ?? graphValue.tool_call_id ?? fallback.toolCallId, + ); + const operation = stringValue( + graphValue.tool ?? graphValue.operation ?? fallback.operation, ); - const operation = stringValue(graphValue.tool ?? graphValue.operation); const childrenValue = graphValue.childRuns ?? graphValue.child_runs; if (!toolCallId || !operation || !Array.isArray(childrenValue)) return undefined; diff --git a/src/web-server.ts b/src/web-server.ts index 551dbcdb6..05fe34ff8 100644 --- a/src/web-server.ts +++ b/src/web-server.ts @@ -72,7 +72,12 @@ import { HostedRunnerDrainStatusValue, drainHostedRunnerForShutdown, } from "./server/handlers/hosted-runner-drain.js"; -import { createWorkspaceConfigMiddleware } from "./services/workspace-config/middleware.js"; +import { + createWorkspaceConfigMiddleware, + loadWorkspaceConfigRequestContext, +} from "./services/workspace-config/middleware.js"; +import { WorkspaceConfigValidationError } from "./services/workspace-config/normalize.js"; +import type { WorkspaceConfigRequestContext } from "./services/workspace-config/types.js"; import { recordApiRequest } from "./telemetry.js"; import { artifactsClientTool } from "./tools/artifacts-client.js"; import { askUserClientTool } from "./tools/ask-user-client.js"; @@ -1034,6 +1039,7 @@ export async function startWebServer( } let runtimeSessionId: string | undefined; + let workspaceConfig: WorkspaceConfigRequestContext | undefined; if (url.pathname === "/api/runtime/ws") { const authorization = await authorizeRuntimeWebSocketSession({ req, @@ -1046,6 +1052,28 @@ export async function startWebServer( return; } runtimeSessionId = authorization; + } else { + try { + workspaceConfig = await loadWorkspaceConfigRequestContext(req); + } catch (error) { + const validationError = error instanceof WorkspaceConfigValidationError; + const status = validationError ? 400 : 503; + const message = validationError + ? error.message + : "Workspace config could not be loaded."; + if (!validationError) { + logger.warn("Failed to load workspace config for WebSocket chat", { + error: sanitizeWithStaticMask( + error instanceof Error ? error.message : String(error), + ), + }); + } + socket.write( + `HTTP/1.1 ${status} ${status === 400 ? "Bad Request" : "Service Unavailable"}\r\nConnection: close\r\nContent-Type: text/plain\r\nContent-Length: ${Buffer.byteLength(message)}\r\n\r\n${message}`, + ); + socket.destroy(); + return; + } } wsServer.handleUpgrade(req, socket, head, (ws) => { @@ -1057,7 +1085,7 @@ export async function startWebServer( }); return; } - handleChatWebSocket(ws, req, context); + handleChatWebSocket(ws, req, context, workspaceConfig); }); }); diff --git a/test/agent/oracle-consultation-policy.test.ts b/test/agent/oracle-consultation-policy.test.ts new file mode 100644 index 000000000..30f53d7ae --- /dev/null +++ b/test/agent/oracle-consultation-policy.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from "vitest"; +import { + ORACLE_CONSULTATION_POLICY_VERSION, + applyOracleConsultationDirective, + formatOracleConsultationDirective, + recommendOracleConsultation, +} from "../../src/agent/oracle-consultation-policy.js"; +import type { AgentProfileLevel } from "../../src/agent/profiles.js"; + +const evalCases: Array<{ + name: string; + profileLevel: AgentProfileLevel; + taskType: string; + taskSummary?: string; + priorFailures?: number; + expected: "available" | "recommended" | "required"; +}> = [ + { + name: "low-risk edit avoids mandatory spend", + profileLevel: "low", + taskType: "coding", + taskSummary: "Rename a local variable", + expected: "available", + }, + { + name: "medium ordinary work keeps oracle available", + profileLevel: "medium", + taskType: "coding", + taskSummary: "Add a bounded parser test", + expected: "available", + }, + { + name: "architecture gets independent review", + profileLevel: "medium", + taskType: "architecture", + taskSummary: "Choose a durable event model", + expected: "recommended", + }, + { + name: "ambiguity triggers consultation", + profileLevel: "medium", + taskType: "coding", + taskSummary: "Requirements are ambiguous with several tradeoffs", + expected: "recommended", + }, + { + name: "migration cues in ordinary chat trigger consultation", + profileLevel: "medium", + taskType: "chat", + taskSummary: "Migrate the session store to the new schema", + expected: "recommended", + }, + { + name: "security cues in ordinary chat trigger consultation", + profileLevel: "medium", + taskType: "chat", + taskSummary: "Review authentication boundaries for this endpoint", + expected: "recommended", + }, + { + name: "architecture cues in ordinary chat trigger consultation", + profileLevel: "medium", + taskType: "chat", + taskSummary: "Redesign the event architecture across services", + expected: "recommended", + }, + { + name: "high profile consults by default", + profileLevel: "high", + taskType: "coding", + taskSummary: "Change routing behavior across packages", + expected: "recommended", + }, + { + name: "ultra profile requires consultation", + profileLevel: "ultra", + taskType: "migration", + taskSummary: "Migrate persisted sessions without data loss", + expected: "required", + }, + { + name: "repeated failures escalate medium work", + profileLevel: "medium", + taskType: "debugging", + priorFailures: 2, + expected: "required", + }, + { + name: "low profile caps repeated-failure escalation", + profileLevel: "low", + taskType: "debugging", + priorFailures: 2, + expected: "recommended", + }, +]; + +describe("oracle consultation policy eval matrix", () => { + for (const evalCase of evalCases) { + it(evalCase.name, () => { + const decision = recommendOracleConsultation(evalCase); + expect(decision.mode).toBe(evalCase.expected); + expect(decision.policyVersion).toBe(ORACLE_CONSULTATION_POLICY_VERSION); + expect(decision.evalSuite).toBe("oracle-consultation-policy-v1"); + }); + } + + it("formats an explicit, read-only required directive", () => { + const directive = formatOracleConsultationDirective( + recommendOracleConsultation({ + profileLevel: "ultra", + taskType: "architecture", + }), + ); + + expect(directive).toContain("MUST consult the read-only Oracle once"); + expect(directive).toContain(ORACLE_CONSULTATION_POLICY_VERSION); + }); + + it("queues recommended guidance exactly once for the next run", () => { + const additions: string[] = []; + const queued = applyOracleConsultationDirective( + { queueNextRunSystemPromptAddition: (text) => additions.push(text) }, + recommendOracleConsultation({ + profileLevel: "high", + taskType: "coding", + }), + ); + + expect(queued).toBe(true); + expect(additions).toHaveLength(1); + expect(additions[0]).toContain("read-only Oracle once"); + }); +}); diff --git a/test/services/intelligent-router.test.ts b/test/services/intelligent-router.test.ts index 1bbc0e079..ba75f4f08 100644 --- a/test/services/intelligent-router.test.ts +++ b/test/services/intelligent-router.test.ts @@ -7,6 +7,7 @@ import { resolveIntelligentRouterStrategy, setIntelligentRouterServiceForTest, } from "../../src/services/intelligent-router/index.js"; +import { resolveIntelligentRouterTaskSummary } from "../../src/services/intelligent-router/recorder.js"; import type { RoutingModelCandidate } from "../../src/services/intelligent-router/types.js"; interface MockResponse { @@ -76,6 +77,17 @@ function createService(): IntelligentRouterService { } describe("intelligent router service", () => { + it("derives policy signals from the latest user message", () => { + expect( + resolveIntelligentRouterTaskSummary({ + messages: [ + { role: "user", content: "Investigate the ambiguous migration" }, + { role: "assistant", content: "A stale assistant continuation" }, + ], + }), + ).toBe("Investigate the ambiguous migration"); + }); + afterEach(() => { setIntelligentRouterServiceForTest(null); vi.restoreAllMocks(); @@ -124,6 +136,22 @@ describe("intelligent router service", () => { }); }); + it("records an eval-backed Oracle consultation decision", () => { + const service = createService(); + + const decision = service.routeRequest({ + taskType: "migration", + profileHint: "ultra", + taskSummary: "Migrate durable sessions without data loss", + }); + + expect(decision.oracleConsultation).toMatchObject({ + policyVersion: "evalops.maestro.oracle-consultation.v1", + evalSuite: "oracle-consultation-policy-v1", + mode: "required", + }); + }); + it("preserves colon-delimited model hints until enough production history exists", () => { const service = createService(); diff --git a/test/session/agent-lineage-projection.test.ts b/test/session/agent-lineage-projection.test.ts index 37b608e24..fd9b5700e 100644 --- a/test/session/agent-lineage-projection.test.ts +++ b/test/session/agent-lineage-projection.test.ts @@ -26,6 +26,41 @@ function graph(toolCallId: string, tool: string, status: string) { } describe("agent lineage projection", () => { + it("falls back to assistant tool-call metadata for sparse work graphs", () => { + const sparseGraph = graph("wrapper-call", "spawnAgent", "spawned"); + delete (sparseGraph as { toolCallId?: string }).toolCallId; + delete (sparseGraph as { tool?: string }).tool; + const entries = [ + { + type: "message", + id: "assistant-sparse-spawn", + parentId: null, + timestamp: "2026-01-01T00:00:01.000Z", + message: { + role: "assistant", + content: [ + { + type: "toolCall", + id: "wrapper-call", + name: "codex.subagent.spawnAgent", + arguments: { codexWorkGraph: sparseGraph }, + }, + ], + }, + }, + ] as SessionTreeEntry[]; + + const projection = buildAgentLineageProjection(entries); + + expect(projection.operations).toEqual([ + expect.objectContaining({ + toolCallId: "wrapper-call", + operation: "spawnAgent", + childRunId: "child-run-1", + }), + ]); + }); + it("aggregates child lifecycle operations into one durable edge", () => { const entries = [ { diff --git a/test/web/chat-handler-profile.test.ts b/test/web/chat-handler-profile.test.ts index 50384c220..b036a928c 100644 --- a/test/web/chat-handler-profile.test.ts +++ b/test/web/chat-handler-profile.test.ts @@ -211,4 +211,109 @@ describe("chat handler profile threading", () => { ); }); }); + + it("applies required Oracle guidance to websocket chat", async () => { + const runUserPromptWithRecovery = vi.fn(async () => {}); + const { handleChatWebSocket } = await importChatHandlersWithMock( + runUserPromptWithRecovery, + ); + const req = new PassThrough() as MockPassThrough; + req.method = "GET"; + req.url = "/api/chat/ws"; + req.headers = { + host: "localhost", + "x-maestro-agent-profile": "ultra", + }; + const ws = new MockWebSocket(); + const queueNextRunSystemPromptAddition = vi.fn(); + const agent = createMockAgent(); + agent.queueNextRunSystemPromptAddition = queueNextRunSystemPromptAddition; + const context: Partial = { + createAgent: async () => agent, + getRegisteredModel: async () => mockModel, + defaultApprovalMode: "prompt", + defaultProvider: "anthropic", + defaultModelId: mockModel.id, + corsHeaders: cors, + }; + + handleChatWebSocket( + ws as unknown as Parameters[0], + req as unknown as IncomingMessage, + context as WebServerContext, + ); + ws.emit( + "message", + JSON.stringify({ + messages: [{ role: "user", content: "Plan a risky migration" }], + }), + ); + + await vi.waitFor(() => { + expect(queueNextRunSystemPromptAddition).toHaveBeenCalledWith( + expect.stringContaining("MUST consult the read-only Oracle once"), + ); + }); + }); + + it("rejects workspace-blocked websocket models and releases the stream lease", async () => { + const runUserPromptWithRecovery = vi.fn(async () => {}); + const { handleChatWebSocket } = await importChatHandlersWithMock( + runUserPromptWithRecovery, + ); + const req = new PassThrough() as MockPassThrough; + req.method = "GET"; + req.url = "/api/chat/ws"; + req.headers = { host: "localhost" }; + const ws = new MockWebSocket(); + const lease = Symbol("websocket-stream"); + const releaseSse = vi.fn(); + const createAgent = vi.fn(async () => createMockAgent()); + const context: Partial = { + createAgent, + getRegisteredModel: async () => mockModel, + defaultApprovalMode: "prompt", + defaultProvider: "anthropic", + defaultModelId: mockModel.id, + corsHeaders: cors, + acquireSse: () => lease, + releaseSse, + }; + + handleChatWebSocket( + ws as unknown as Parameters[0], + req as unknown as IncomingMessage, + context as WebServerContext, + { + workspaceId: "workspace-a", + source: "database", + config: { + workspaceId: "workspace-a", + modelPreferences: { + allowedModels: [], + blockedModels: [`${mockModel.provider}/${mockModel.id}`], + }, + safetyRules: { + allowedTools: [], + blockedTools: [], + requiredSkills: [], + fileBoundaries: [], + }, + rateLimits: {}, + createdAt: "2026-07-14T00:00:00.000Z", + updatedAt: "2026-07-14T00:00:00.000Z", + }, + }, + ); + ws.emit( + "message", + JSON.stringify({ messages: [{ role: "user", content: "Hello" }] }), + ); + + await vi.waitFor(() => { + expect(releaseSse).toHaveBeenCalledWith(lease); + }); + expect(createAgent).not.toHaveBeenCalled(); + expect(runUserPromptWithRecovery).not.toHaveBeenCalled(); + }); }); diff --git a/test/web/chat-handler-routing.test.ts b/test/web/chat-handler-routing.test.ts index 185bd0048..58bfa2668 100644 --- a/test/web/chat-handler-routing.test.ts +++ b/test/web/chat-handler-routing.test.ts @@ -16,6 +16,7 @@ import { resolveAgentProfile } from "../../src/agent/profiles.js"; import type { RegisteredModel } from "../../src/models/registry.js"; import type { WebServerContext } from "../../src/server/app-context.js"; import { handleChat } from "../../src/server/handlers/chat.js"; +import { resolveModelInputForRouting } from "../../src/server/model-selection.js"; import { requestContextStorage } from "../../src/server/request-context.js"; import { selectIntelligentRouterModel } from "../../src/services/intelligent-router/recorder.js"; import type { RoutingDecision } from "../../src/services/intelligent-router/types.js"; @@ -202,5 +203,14 @@ describe("handleChat routing", () => { expect(res.statusCode).toBe(500); expect(res.body).toContain("Missing API key for anthropic"); expect(res.body).not.toContain("not allowed by workspace policy"); + expect(selectIntelligentRouterModel).toHaveBeenCalledWith( + expect.objectContaining({ + requestedModel: resolveModelInputForRouting( + undefined, + selectedModel.provider, + selectedModel.id, + ), + }), + ); }); }); diff --git a/test/web/model-selection.test.ts b/test/web/model-selection.test.ts index d7450039c..f8818832b 100644 --- a/test/web/model-selection.test.ts +++ b/test/web/model-selection.test.ts @@ -37,6 +37,7 @@ import { determineModelSelection, getRegisteredModelOrThrow, parseModelInput, + resolveModelInputForRouting, } from "../../src/server/model-selection.js"; describe("model-selection", () => { @@ -68,6 +69,15 @@ describe("model-selection", () => { }); }); + it("normalizes aliases and defaults into router model hints", () => { + expect( + resolveModelInputForRouting("fast-model", "anthropic", "claude-3"), + ).toBe("openai/gpt-fast"); + expect(resolveModelInputForRouting(undefined, "openai", "gpt-4o")).toBe( + "anthropic/claude-default", + ); + }); + it("preserves registered providers for bare model ids", () => { const selection = determineModelSelection( "claude-default",