chore: sync public mirror from internal

docs/superpowers/plans/2026-07-13-automatic-oracle-policy.md

@@ -0,0 +1,35 @@
+# Automatic Oracle Consultation Policy Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Automatically and audibly recommend or require the read-only Oracle for tasks where independent reasoning is likely to improve outcomes.
+
+**Architecture:** A pure versioned policy evaluates profile, task type, prompt signals, and prior failures. The intelligent router records the decision, and the chat handler injects explicit next-run guidance; a checked-in eval matrix prevents trigger drift.
+
+**Tech Stack:** TypeScript, Vitest, intelligent router, Agent system-prompt additions.
+
+## Global Constraints
+
+- No hidden tool execution; the agent sees and follows an explicit policy directive.
+- Oracle remains read-only and uses the profile’s complementary model.
+- Low-risk profiles avoid mandatory Oracle cost.
+- Policy behavior must be covered by named eval cases before integration.
+
+---
+
+### Task 1: Versioned consultation policy
+
+**Files:** Create `src/agent/oracle-consultation-policy.ts`; test `test/agent/oracle-consultation-policy.test.ts`.
+
+- [x] Write a failing eval matrix for low, medium, high, ultra, architecture/migration, ambiguity, cross-cutting work, and repeated failures.
+- [x] Implement minimal deterministic scoring and prompt directive formatting.
+- [x] Verify all eval cases pass.
+
+### Task 2: Router and runtime integration
+
+**Files:** Modify intelligent-router types/normalization/service/recorder and `src/server/handlers/chat.ts`; update focused tests.
+
+- [x] Add failing tests for decision recording and next-run prompt injection.
+- [x] Thread task summary and prior failure count through routing.
+- [x] Queue policy guidance only for recommended/required modes.
+- [ ] Run targeted tests, lint, affected tests, commit, open PR after PR 2 merges, address feedback, merge, and verify deployment/mirror workflows.

src/agent/oracle-consultation-policy.ts

@@ -0,0 +1,110 @@
+import type { AgentProfileLevel } from "./profiles.js";
+
+export const ORACLE_CONSULTATION_POLICY_VERSION =
+	"evalops.maestro.oracle-consultation.v1";
+
+export type OracleConsultationMode = "available" | "recommended" | "required";
+
+export interface OracleConsultationPolicyInput {
+	profileLevel: AgentProfileLevel;
+	taskType: string;
+	taskSummary?: string;
+	priorFailures?: number;
+}
+
+export interface OracleConsultationDecision {
+	policyVersion: typeof ORACLE_CONSULTATION_POLICY_VERSION;
+	evalSuite: "oracle-consultation-policy-v1";
+	mode: OracleConsultationMode;
+	reasons: string[];
+}
+
+const CONSULTATION_TASK_TYPES = new Set([
+	"architecture",
+	"code_review",
+	"discovery",
+	"incident_response",
+	"migration",
+	"planning",
+	"security_review",
+]);
+
+const UNCERTAINTY_PATTERN =
+	/\b(?:ambiguous|unclear|uncertain|trade-?offs?|cross[- ]cutting|multiple approaches|data loss|irreversible|root cause unknown)\b/i;
+
+const CONSULTATION_PROMPT_PATTERN =
+	/\b(?:architect(?:ure|ural|ing)?|migrat(?:e|es|ed|ing|ion)|security|secure|threat model(?:ing)?|auth(?:entication|orization)?|schema (?:change|migration)|backfill)\b/i;
+
+export function recommendOracleConsultation(
+	input: OracleConsultationPolicyInput,
+): OracleConsultationDecision {
+	const reasons: string[] = [];
+	const priorFailures = Math.max(0, Math.floor(input.priorFailures ?? 0));
+	let mode: OracleConsultationMode = "available";
+
+	if (input.profileLevel === "ultra") {
+		mode = "required";
+		reasons.push("ultra_profile");
+	} else if (input.profileLevel === "high") {
+		mode = "recommended";
+		reasons.push("high_profile");
+	}
+
+	if (CONSULTATION_TASK_TYPES.has(input.taskType.trim().toLowerCase())) {
+		if (mode === "available") mode = "recommended";
+		reasons.push("consultation_task_type");
+	}
+
+	if (input.taskSummary && UNCERTAINTY_PATTERN.test(input.taskSummary)) {
+		if (mode === "available") mode = "recommended";
+		reasons.push("uncertainty_signal");
+	}
+
+	if (
+		input.taskSummary &&
+		CONSULTATION_PROMPT_PATTERN.test(input.taskSummary)
+	) {
+		if (mode === "available") mode = "recommended";
+		reasons.push("consultation_prompt_signal");
+	}
+
+	if (priorFailures >= 2) {
+		mode = input.profileLevel === "low" ? "recommended" : "required";
+		reasons.push("repeated_failures");
+	}
+
+	if (reasons.length === 0) reasons.push("oracle_available_on_demand");
+	return {
+		policyVersion: ORACLE_CONSULTATION_POLICY_VERSION,
+		evalSuite: "oracle-consultation-policy-v1",
+		mode,
+		reasons,
+	};
+}
+
+export function formatOracleConsultationDirective(
+	decision: OracleConsultationDecision,
+): string {
+	const instruction =
+		decision.mode === "required"
+			? "You MUST consult the read-only Oracle once before committing to the plan or final answer. Incorporate or explicitly rebut its advice."
+			: decision.mode === "recommended"
+				? "Consult the read-only Oracle once before committing to the plan or final answer unless the task has become clearly bounded; if you skip it, state the concrete reason."
+				: "The read-only Oracle is available on demand.";
+	return [
+		`Oracle consultation policy (${decision.policyVersion})`,
+		instruction,
+		`Triggers: ${decision.reasons.join(", ")}.`,
+	].join("\n");
+}
+
+export function applyOracleConsultationDirective(
+	agent: { queueNextRunSystemPromptAddition(text: string): void },
+	decision: OracleConsultationDecision | undefined,
+): boolean {
+	if (!decision || decision.mode === "available") return false;
+	agent.queueNextRunSystemPromptAddition(
+		formatOracleConsultationDirective(decision),
+	);
+	return true;
+}

src/server/handlers/chat-ws.ts

@@ -4,9 +4,11 @@
  * Mirrors the SSE chat flow but streams AgentEvent payloads over WebSocket.
  */
 
+import { randomBytes, randomUUID } from "node:crypto";
 import type { IncomingMessage } from "node:http";
 import type { ComposerChatRequest, ComposerMessage } from "@evalops/contracts";
 import type { RawData, WebSocket } from "ws";
+import { applyOracleConsultationDirective } from "../../agent/oracle-consultation-policy.js";
 import { isAssistantMessage } from "../../agent/type-guards.js";
 import type {
 	Attachment as AgentAttachment,
@@ -22,7 +24,13 @@ import {
 } from "../../memory/auto-consolidation.js";
 import { createAutomaticMemoryExtractionCoordinator } from "../../memory/auto-extraction.js";
 import type { RegisteredModel } from "../../models/registry.js";
+import {
+	recordIntelligentRouterChatMetric,
+	selectIntelligentRouterModel,
+} from "../../services/intelligent-router/recorder.js";
 import { recordAssistantUsageMetric } from "../../services/usage-analytics/recorder.js";
+import { evaluateModelPolicy } from "../../services/workspace-config/policy.js";
+import type { WorkspaceConfigRequestContext } from "../../services/workspace-config/types.js";
 import { toSessionModelMetadata } from "../../session/manager.js";
 import { createRuntimeSessionSummaryUpdater } from "../../session/runtime-summary-updater.js";
 import { createLogger } from "../../utils/logger.js";
@@ -37,6 +45,13 @@ import { getAuthSubject } from "../authz.js";
 import { getAgentCircuitBreaker } from "../circuit-breaker.js";
 import { clientToolService } from "../client-tools-service.js";
 import { isHostedSessionManager } from "../hosted-session-manager.js";
+import { resolveModelInputForRouting } from "../model-selection.js";
+import {
+	type RequestContext,
+	getWorkspaceConfigContext,
+	parseTraceParent,
+	requestContextStorage,
+} from "../request-context.js";
 import { serverRequestManager } from "../server-request-manager.js";
 import { getRequestHeader } from "../server-utils.js";
 import { startSessionWithPolicy } from "../session-initialization.js";
@@ -219,12 +234,15 @@ export function handleChatWebSocket(
 	ws: WebSocket,
 	req: IncomingMessage,
 	context: WebServerContext,
+	workspaceConfig?: WorkspaceConfigRequestContext,
 ) {
 	const {
 		createAgent,
 		createBackgroundAgent,
 		getRegisteredModel,
 		defaultApprovalMode,
+		defaultProvider,
+		defaultModelId,
 		acquireSse,
 		releaseSse,
 	} = context;
@@ -245,6 +263,17 @@ export function handleChatWebSocket(
 	);
 	const slimFromQuery = parseBoolean(url.searchParams.get("slim"));
 	const clientHeaderFromQuery = url.searchParams.get("client")?.trim();
+	const websocketRequestContext: RequestContext | undefined = workspaceConfig
+		? {
+				requestId: randomUUID(),
+				traceId: parseTraceParent(req.headers.traceparent).traceId,
+				spanId: randomBytes(8).toString("hex"),
+				startTime: performance.now(),
+				method: req.method || "GET",
+				url: url.pathname,
+				workspaceConfig,
+			}
+		: undefined;
 
 	const sendErrorAndClose = (message: string) => {
 		try {
@@ -271,7 +300,7 @@ export function handleChatWebSocket(
 		return validatePayload<ChatRequestInput>(parsed, ChatRequestSchema);
 	};
 
-	ws.on("message", async (data) => {
+	const handleMessage = async (data: RawData) => {
 		if (requestHandled) {
 			try {
 				const size = getRawDataSize(data);
@@ -450,7 +479,55 @@ export function handleChatWebSocket(
 				}
 			}
 
-			const registeredModel = await getRegisteredModel(chatReq.model);
+			const routingSelection = selectIntelligentRouterModel({
+				req,
+				requestedModel: resolveModelInputForRouting(
+					chatReq.model,
+					defaultProvider,
+					defaultModelId,
+				),
+				body: chatReq,
+			});
+			let registeredModel: RegisteredModel | undefined;
+			let lastModelError: unknown;
+			let selectedModelError: unknown;
+			let selectedModelPolicyViolation: ReturnType<
+				typeof evaluateModelPolicy
+			> | null = null;
+			for (const [
+				index,
+				modelInput,
+			] of routingSelection.modelInputs.entries()) {
+				try {
+					const candidateModel = await getRegisteredModel(modelInput);
+					const violation = evaluateModelPolicy(
+						workspaceConfig?.config ?? getWorkspaceConfigContext()?.config,
+						{
+							provider: candidateModel.provider,
+							modelId: candidateModel.id,
+						},
+					);
+					if (violation) {
+						if (index === 0) selectedModelPolicyViolation = violation;
+						continue;
+					}
+					registeredModel = candidateModel;
+					break;
+				} catch (error) {
+					lastModelError = error;
+					if (index === 0) selectedModelError = error;
+				}
+			}
+			if (!registeredModel && selectedModelPolicyViolation) {
+				sendErrorAndClose(selectedModelPolicyViolation.message);
+				if (sseLease && releaseSse) {
+					releaseSse(sseLease);
+					sseLease = null;
+				}
+				return;
+			}
+			if (!registeredModel) throw selectedModelError ?? lastModelError;
+			const routeStartedAt = Date.now();
 
 			const headerApproval = (() => {
 				const headerMode = normalizeApprovalMode(
@@ -529,6 +606,10 @@ export function handleChatWebSocket(
 						: {}),
 				},
 			);
+			applyOracleConsultationDirective(
+				agent,
+				routingSelection.decision.oracleConsultation,
+			);
 
 			const historyMessages = incomingMessages.slice(0, -1);
 			const hydratedHistory = convertComposerMessagesToApp(
@@ -844,6 +925,13 @@ export function handleChatWebSocket(
 							sessionId: sessionManager.getSessionId(),
 							subject,
 						});
+						recordIntelligentRouterChatMetric({
+							taskType: routingSelection.taskType,
+							provider: registeredModel.provider,
+							model: registeredModel.id,
+							startedAt: routeStartedAt,
+							message: event.message,
+						});
 						automaticMemoryExtraction.schedule(sessionManager.getSessionFile());
 					}
 					sessionManager.updateSnapshot(
@@ -962,5 +1050,14 @@ export function handleChatWebSocket(
 				sseLease = null;
 			}
 		}
+	};
+
+	ws.on("message", (data) => {
+		if (websocketRequestContext) {
+			return requestContextStorage.run(websocketRequestContext, () =>
+				handleMessage(data),
+			);
+		}
+		return handleMessage(data);
 	});
 }

src/server/handlers/chat.ts

@@ -23,6 +23,7 @@
 
 import type { IncomingMessage, ServerResponse } from "node:http";
 import type { ComposerChatRequest, ComposerMessage } from "@evalops/contracts";
+import { applyOracleConsultationDirective } from "../../agent/oracle-consultation-policy.js";
 import { isAssistantMessage } from "../../agent/type-guards.js";
 import type {
 	Attachment as AgentAttachment,
@@ -63,6 +64,7 @@ import { getAuthSubject } from "../authz.js";
 import { getAgentCircuitBreaker } from "../circuit-breaker.js";
 import { clientToolService } from "../client-tools-service.js";
 import { isHostedSessionManager } from "../hosted-session-manager.js";
+import { resolveModelInputForRouting } from "../model-selection.js";
 import { getWorkspaceConfigContext } from "../request-context.js";
 import { serverRequestManager } from "../server-request-manager.js";
 import {
@@ -131,6 +133,8 @@ export async function handleChat(
 		createBackgroundAgent,
 		getRegisteredModel,
 		defaultApprovalMode,
+		defaultProvider,
+		defaultModelId,
 		acquireSse,
 		releaseSse,
 		corsHeaders: cors,
@@ -319,7 +323,11 @@ export async function handleChat(
 		// it can select a better-scoring model and expose explicit fallbacks.
 		const routingSelection = selectIntelligentRouterModel({
 			req,
-			requestedModel: chatReq.model,
+			requestedModel: resolveModelInputForRouting(
+				chatReq.model,
+				defaultProvider,
+				defaultModelId,
+			),
 			body: chatReq,
 		});
 		let registeredModel: RegisteredModel | undefined;
@@ -434,6 +442,10 @@ export async function handleChat(
 					: {}),
 			},
 		);
+		applyOracleConsultationDirective(
+			agent,
+			routingSelection.decision.oracleConsultation,
+		);
 
 		// Hydrate conversation history (all messages except the current user input)
 		const historyMessages = incomingMessages.slice(0, -1);

src/server/model-selection.ts

@@ -130,6 +130,19 @@ export function determineModelSelection(
 	};
 }
 
+export function resolveModelInputForRouting(
+	modelInput: string | null | undefined,
+	defaultProvider: string,
+	defaultModelId: string,
+): string {
+	const selection = determineModelSelection(
+		modelInput,
+		defaultProvider,
+		defaultModelId,
+	);
+	return `${selection.provider}/${selection.modelId}`;
+}
+
 export function getRegisteredModelOrThrow(
 	selection: ModelSelection,
 ): RegisteredModel {